diff --git a/.gitignore b/.gitignore index 09734fe4974935956fd599f7f86cd5c4d195d5e2..9ae0d9c96f188bc6357832f22b4125694302b104 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,8 @@ cmake_build/ .idea/** /build/ /tensorflow/core/util/version_info.cc +/tensorflow/python/framework/fast_tensor_util.cpp +Pods +Podfile.lock +*.pbxproj +*.xcworkspacedata diff --git a/RELEASE.md b/RELEASE.md index d30ee69f40e24672f1d68f81109e5d9bd266e81d..d8db1f72004b5d944e3035a0f33dfc34a674b7ee 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,7 +2,8 @@ ## Major Features And Improvements * `tf.keras` is now part of the core TensorFlow API. -* `tf.data` is now part of the core TensorFlow API. +* [`tf.data`](http://tensorflow.org/programmers_guide/datasets) is now part of + the core TensorFlow API. * The API is now subject to backwards compatibility guarantees. * For a guide to migrating from the `tf.contrib.data` API, see the [README](https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/contrib/data/README.md). @@ -18,6 +19,14 @@ (with GPU and gradient support). * Add a self-check on `import tensorflow` for Windows DLL issues. * Add NCHW support to `tf.depth_to_space` on GPU. +* TensorFlow Debugger (tfdbg): + * Add `eval` command to allow evaluation of arbitrary Python/numpy expressions + in tfdbg command-line interface. See + [Debugging TensorFlow Programs](https://www.tensorflow.org/programmers_guide/debugger) + for more details. + * Usability improvement: The frequently used tensor filter `has_inf_or_nan` is + now added to `Session` wrappers and hooks by default. So there is no need + for clients to call `.add_tensor_filter(tf_debug.has_inf_or_nan)` anymore. * SinhArcsinh (scalar) distribution added to `contrib.distributions`. * Make `GANEstimator` opensource. * `Estimator.export_savedmodel()` now includes all valid serving signatures @@ -59,10 +68,14 @@ * Fix `tf.contrib.distributions.Affine` incorrectly computing log-det-jacobian. * Fix `tf.random_gamma` incorrectly handling non-batch, scalar draws. * Resolved a race condition in TensorForest TreePredictionsV4Op. -* Google Cloud Storage file system and Hadoop file system support are now - default build options. +* Google Cloud Storage file system, Amazon S3 file system, and Hadoop file + system support are now default build options. * Custom op libraries must link against libtensorflow_framework.so (installed at `tf.sysconfig.get_lib()`). +* Change `RunConfig` default behavior to not set a random seed, making random + behavior independently random on distributed workers. We expect this to + generally improve training performance. Models that do rely on determinism + should set a random seed explicitly. ## Breaking Changes to the API * The signature of the `tf.contrib.data.rejection_resample()` function has been @@ -73,6 +86,11 @@ * Remove seldom used and unnecessary `tf.contrib.data.Iterator.dispose_op()`. * Reorder some TFGAN loss functions in a non-backwards compatible way. +## Known Issues +* In Python 3, `Dataset.from_generator()` does not support Unicode strings. + You must convert any strings to bytes objects before yielding them from + the generator. + ## Thanks to our Contributors This release contains contributions from many people at Google, as well as: diff --git a/WORKSPACE b/WORKSPACE index 1bf1069f8801c9d135d77c871520ff733b7713e9..b40913801ba8e3c8ee73f7ba69540b520ad698a6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,7 +5,7 @@ http_archive( sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257", strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", # 2017-08-28 ], ) diff --git a/configure.py b/configure.py index 95835e538b62371d671aa7adb0f2f12b71639a58..bc7859fee4d2aca9bd7ca24e85ad820c49e01e4a 100644 --- a/configure.py +++ b/configure.py @@ -963,6 +963,19 @@ def set_monolithic(): write_to_bazelrc('build --define framework_shared_object=true') +def create_android_bazelrc_configs(): + # Flags for --config=android + write_to_bazelrc('build:android --crosstool_top=//external:android/crosstool') + write_to_bazelrc( + 'build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain') + # Flags for --config=android_arm + write_to_bazelrc('build:android_arm --config=android') + write_to_bazelrc('build:android_arm --cpu=armeabi-v7a') + # Flags for --config=android_arm64 + write_to_bazelrc('build:android_arm64 --config=android') + write_to_bazelrc('build:android_arm64 --cpu=arm64-v8a') + + def main(): # Make a copy of os.environ to be clear when functions and getting and setting # environment variables. @@ -981,6 +994,7 @@ def main(): environ_cp['TF_NEED_HDFS'] = '0' environ_cp['TF_NEED_JEMALLOC'] = '0' environ_cp['TF_NEED_OPENCL'] = '0' + environ_cp['TF_NEED_S3'] = '0' environ_cp['TF_CUDA_CLANG'] = '0' if is_macos(): @@ -1033,7 +1047,7 @@ def main(): set_cc_opt_flags(environ_cp) set_mkl() set_monolithic() - + create_android_bazelrc_configs() if __name__ == '__main__': main() diff --git a/tensorflow/BUILD b/tensorflow/BUILD index fa5da5fdbb4442ea1b971623ea1447ddd2e8f4d6..3f23203aefd3d42c12c6a40f3711bcdedd22fd23 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -123,7 +123,7 @@ config_setting( config_setting( name = "ios_x86_64", values = { - "cc_target_os": "apple", + "crosstool_top": "//tools/osx/crosstool:crosstool", "cpu": "ios_x86_64", }, visibility = ["//visibility:public"], @@ -331,6 +331,7 @@ filegroup( "//tensorflow/compiler/jit/kernels:all_files", "//tensorflow/compiler/jit/legacy_flags:all_files", "//tensorflow/compiler/jit/ops:all_files", + "//tensorflow/compiler/plugin:all_files", "//tensorflow/compiler/tests:all_files", "//tensorflow/compiler/tf2xla:all_files", "//tensorflow/compiler/tf2xla/cc:all_files", @@ -348,6 +349,7 @@ filegroup( "//tensorflow/compiler/xla/service/llvm_ir:all_files", "//tensorflow/compiler/xla/tests:all_files", "//tensorflow/compiler/xla/tools:all_files", + "//tensorflow/compiler/xla/tools/parser:all_files", "//tensorflow/contrib:all_files", "//tensorflow/contrib/all_reduce:all_files", "//tensorflow/contrib/android:all_files", @@ -369,6 +371,7 @@ filegroup( "//tensorflow/contrib/crf:all_files", "//tensorflow/contrib/cudnn_rnn:all_files", "//tensorflow/contrib/data:all_files", + "//tensorflow/contrib/data/kernels:all_files", "//tensorflow/contrib/data/python/kernel_tests:all_files", "//tensorflow/contrib/data/python/ops:all_files", "//tensorflow/contrib/decision_trees/proto:all_files", @@ -407,6 +410,7 @@ filegroup( "//tensorflow/contrib/linear_optimizer:all_files", "//tensorflow/contrib/lookup:all_files", "//tensorflow/contrib/losses:all_files", + "//tensorflow/contrib/makefile:all_files", "//tensorflow/contrib/meta_graph_transform:all_files", "//tensorflow/contrib/metrics:all_files", "//tensorflow/contrib/mpi_collectives:all_files", @@ -421,7 +425,6 @@ filegroup( "//tensorflow/contrib/remote_fused_graph/pylib:all_files", "//tensorflow/contrib/resampler:all_files", "//tensorflow/contrib/rnn:all_files", - "//tensorflow/contrib/s3:all_files", "//tensorflow/contrib/saved_model:all_files", "//tensorflow/contrib/saved_model/cc/saved_model:all_files", "//tensorflow/contrib/seq2seq:all_files", @@ -443,6 +446,7 @@ filegroup( "//tensorflow/contrib/tensor_forest/kernels/v4:all_files", "//tensorflow/contrib/tensor_forest/proto:all_files", "//tensorflow/contrib/tensorboard:all_files", + "//tensorflow/contrib/tensorboard/db:all_files", "//tensorflow/contrib/testing:all_files", "//tensorflow/contrib/text:all_files", "//tensorflow/contrib/tfprof:all_files", @@ -455,7 +459,6 @@ filegroup( "//tensorflow/contrib/training:all_files", "//tensorflow/contrib/util:all_files", "//tensorflow/contrib/verbs:all_files", - "//tensorflow/contrib/xla_tf_graph:all_files", "//tensorflow/core:all_files", "//tensorflow/core/debug:all_files", "//tensorflow/core/distributed_runtime:all_files", @@ -475,6 +478,7 @@ filegroup( "//tensorflow/core/platform/cloud:all_files", "//tensorflow/core/platform/default/build_config:all_files", "//tensorflow/core/platform/hadoop:all_files", + "//tensorflow/core/platform/s3:all_files", "//tensorflow/core/profiler:all_files", "//tensorflow/core/profiler/internal:all_files", "//tensorflow/core/profiler/internal/advisor:all_files", @@ -509,6 +513,7 @@ filegroup( "//tensorflow/python/kernel_tests:all_files", "//tensorflow/python/kernel_tests/distributions:all_files", "//tensorflow/python/kernel_tests/linalg:all_files", + "//tensorflow/python/kernel_tests/random:all_files", "//tensorflow/python/ops/distributions:all_files", "//tensorflow/python/ops/linalg:all_files", "//tensorflow/python/profiler:all_files", @@ -519,6 +524,7 @@ filegroup( "//tensorflow/tools/api/golden:all_files", "//tensorflow/tools/api/lib:all_files", "//tensorflow/tools/api/tests:all_files", + "//tensorflow/tools/benchmark:all_files", "//tensorflow/tools/build_info:all_files", "//tensorflow/tools/common:all_files", "//tensorflow/tools/compatibility:all_files", diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 334f867e47800507760eaa71dce91186f646f72d..6dd1b999102d0135720b6ab3a43cbe61255acbc1 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -81,11 +81,13 @@ using tensorflow::TensorBuffer; using tensorflow::TensorId; using tensorflow::TensorShape; using tensorflow::TensorShapeProto; +using tensorflow::VersionDef; using tensorflow::error::Code; using tensorflow::errors::FailedPrecondition; using tensorflow::errors::InvalidArgument; using tensorflow::gtl::ArraySlice; using tensorflow::mutex_lock; +using tensorflow::string; using tensorflow::strings::StrCat; extern "C" { @@ -366,7 +368,7 @@ namespace { // Reset helper for converting character arrays to string vectors. void TF_Reset_Helper(const TF_SessionOptions* opt, const char** containers, int ncontainers, TF_Status* status) { - std::vector container_names(ncontainers); + std::vector container_names(ncontainers); for (int i = 0; i < ncontainers; ++i) { container_names[i] = containers[i]; } @@ -482,7 +484,7 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst) { const char* limit = input + src_size; *dst = Tensor(static_cast(src->dtype), src->shape); - auto dstarray = dst->flat(); + auto dstarray = dst->flat(); for (tensorflow::int64 i = 0; i < num_elements; ++i) { tensorflow::uint64 offset = reinterpret_cast(input)[i]; @@ -556,9 +558,9 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, // Compute bytes needed for encoding. size_t size = 0; - const auto& srcarray = src.flat(); + const auto& srcarray = src.flat(); for (int i = 0; i < srcarray.size(); ++i) { - const tensorflow::string& s = srcarray(i); + const string& s = srcarray(i); // uint64 starting_offset, TF_StringEncode-d string. size += sizeof(tensorflow::uint64) + TF_StringEncodedSize(s.size()); } @@ -572,7 +574,7 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, for (int i = 0; i < srcarray.size(); ++i) { *offsets = (dst - data_start); offsets++; - const tensorflow::string& s = srcarray(i); + const string& s = srcarray(i); size_t consumed = TF_StringEncode(s.data(), s.size(), dst, dst_len, status); if (!status->status.ok()) { status->status = InvalidArgument( @@ -637,10 +639,9 @@ static void TF_Run_Setup(int noutputs, TF_Tensor** c_outputs, } } -static bool TF_Run_Inputs( - TF_Tensor* const* c_inputs, - std::vector>* input_pairs, - TF_Status* status) { +static bool TF_Run_Inputs(TF_Tensor* const* c_inputs, + std::vector>* input_pairs, + TF_Status* status) { const int ninputs = input_pairs->size(); for (int i = 0; i < ninputs; ++i) { status->status = TF_TensorToTensor(c_inputs[i], &(*input_pairs)[i].second); @@ -652,13 +653,12 @@ static bool TF_Run_Inputs( static void TF_Run_Helper( Session* session, const char* handle, const TF_Buffer* run_options, // Input tensors - const std::vector>& input_pairs, + const std::vector>& input_pairs, // Output tensors - const std::vector& output_tensor_names, - TF_Tensor** c_outputs, + const std::vector& output_tensor_names, TF_Tensor** c_outputs, // Target nodes - const std::vector& target_oper_names, - TF_Buffer* run_metadata, TF_Status* status) { + const std::vector& target_oper_names, TF_Buffer* run_metadata, + TF_Status* status) { const int noutputs = output_tensor_names.size(); std::vector outputs(noutputs); Status result; @@ -718,16 +718,16 @@ void TF_Run(TF_DeprecatedSession* s, const TF_Buffer* run_options, const char** c_target_oper_names, int ntargets, TF_Buffer* run_metadata, TF_Status* status) { TF_Run_Setup(noutputs, c_outputs, status); - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(c_inputs, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = c_input_names[i]; } - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = c_output_names[i]; } - std::vector target_oper_names(ntargets); + std::vector target_oper_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_oper_names[i] = c_target_oper_names[i]; } @@ -745,9 +745,9 @@ void TF_PRunSetup(TF_DeprecatedSession* s, const char** handle, TF_Status* status) { *handle = nullptr; - std::vector input_names(ninputs); - std::vector output_names(noutputs); - std::vector target_oper_names(ntargets); + std::vector input_names(ninputs); + std::vector output_names(noutputs); + std::vector target_oper_names(ntargets); for (int i = 0; i < ninputs; ++i) { input_names[i] = c_input_names[i]; } @@ -757,7 +757,7 @@ void TF_PRunSetup(TF_DeprecatedSession* s, for (int i = 0; i < ntargets; ++i) { target_oper_names[i] = c_target_oper_names[i]; } - tensorflow::string new_handle; + string new_handle; status->status = s->session->PRunSetup(input_names, output_names, target_oper_names, &new_handle); if (status->status.ok()) { @@ -776,17 +776,17 @@ void TF_PRun(TF_DeprecatedSession* s, const char* handle, const char** c_target_oper_names, int ntargets, TF_Status* status) { TF_Run_Setup(noutputs, c_outputs, status); - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(c_inputs, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = c_input_names[i]; } - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = c_output_names[i]; } - std::vector target_oper_names(ntargets); + std::vector target_oper_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_oper_names[i] = c_target_oper_names[i]; } @@ -881,7 +881,7 @@ TF_Operation* ToOperation(Node* node) { return static_cast(static_cast(node)); } -tensorflow::string OutputName(const TF_Output& output) { +string OutputName(const TF_Output& output) { return StrCat(output.oper->node.name(), ":", output.index); } @@ -1254,7 +1254,7 @@ void TF_SetAttrValueProto(TF_OperationDescription* desc, const char* attr_name, return; } desc->colocation_constraints.clear(); - for (const tensorflow::string& location : attr_value.list().s()) { + for (const string& location : attr_value.list().s()) { desc->colocation_constraints.insert(location); } } else { @@ -1276,8 +1276,8 @@ static TF_Operation* TF_FinishOperationLocked(TF_OperationDescription* desc, if (!desc->colocation_constraints.empty()) { desc->node_builder.Attr( tensorflow::kColocationAttrName, - std::vector(desc->colocation_constraints.begin(), - desc->colocation_constraints.end())); + std::vector(desc->colocation_constraints.begin(), + desc->colocation_constraints.end())); } status->status = desc->node_builder.Finalize(&desc->graph->graph, &ret); @@ -1500,7 +1500,7 @@ TF_AttrMetadata TF_OperationGetAttrMetadata(TF_Operation* oper, for (int i = 0; i < oper->node.op_def().attr_size(); ++i) { const auto& a = oper->node.op_def().attr(i); if (a.name().compare(attr_name) != 0) continue; - const tensorflow::string& typestr = a.type(); + const string& typestr = a.type(); if (typestr == "list(string)") { metadata.type = TF_ATTR_STRING; } else if (typestr == "list(int)") { @@ -1580,7 +1580,7 @@ void TF_OperationGetAttrStringList(TF_Operation* oper, const char* attr_name, const auto len = std::min(max_values, attr->list().s_size()); char* p = static_cast(storage); for (int i = 0; i < len; ++i) { - const tensorflow::string& s = attr->list().s(i); + const string& s = attr->list().s(i); values[i] = p; lengths[i] = s.size(); if ((p + s.size()) > (static_cast(storage) + storage_size)) { @@ -1799,6 +1799,27 @@ void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def, status->status = MessageToBuffer(def, output_graph_def); } +void TF_GraphGetOpDef(TF_Graph* graph, const char* op_name, + TF_Buffer* output_op_def, TF_Status* status) { + const OpDef* op_def; + { + mutex_lock l(graph->mu); + status->status = graph->graph.op_registry()->LookUpOpDef(op_name, &op_def); + if (!status->status.ok()) return; + } + status->status = MessageToBuffer(*op_def, output_op_def); +} + +void TF_GraphVersions(TF_Graph* graph, TF_Buffer* output_version_def, + TF_Status* status) { + VersionDef versions; + { + mutex_lock l(graph->mu); + versions = graph->graph.versions(); + } + status->status = MessageToBuffer(versions, output_version_def); +} + TF_ImportGraphDefOptions* TF_NewImportGraphDefOptions() { return new TF_ImportGraphDefOptions; } @@ -1813,7 +1834,11 @@ void TF_ImportGraphDefOptionsSetPrefix(TF_ImportGraphDefOptions* opts, void TF_ImportGraphDefOptionsAddInputMapping(TF_ImportGraphDefOptions* opts, const char* src_name, int src_index, TF_Output dst) { - opts->opts.input_map[TensorId(src_name, src_index)] = ToTensorId(dst); + opts->tensor_id_data.push_back(src_name); + const string& src_name_str = opts->tensor_id_data.back(); + // We don't need to store dst's name in tensor_id_data, since `dst` must + // outlive the ImportGraphDef call. + opts->opts.input_map[TensorId(src_name_str, src_index)] = ToTensorId(dst); } void TF_ImportGraphDefOptionsRemapControlDependency( @@ -1829,7 +1854,9 @@ extern void TF_ImportGraphDefOptionsAddControlDependency( void TF_ImportGraphDefOptionsAddReturnOutput(TF_ImportGraphDefOptions* opts, const char* oper_name, int index) { - opts->opts.return_tensors.push_back({oper_name, index}); + opts->tensor_id_data.push_back(oper_name); + const string& oper_name_str = opts->tensor_id_data.back(); + opts->opts.return_tensors.emplace_back(oper_name_str, index); } int TF_ImportGraphDefOptionsNumReturnOutputs( @@ -1837,57 +1864,142 @@ int TF_ImportGraphDefOptionsNumReturnOutputs( return opts->opts.return_tensors.size(); } +void TF_ImportGraphDefOptionsAddReturnOperation(TF_ImportGraphDefOptions* opts, + const char* oper_name) { + opts->opts.return_nodes.push_back(oper_name); +} + +int TF_ImportGraphDefOptionsNumReturnOperations( + const TF_ImportGraphDefOptions* opts) { + return opts->opts.return_nodes.size(); +} + +void TF_ImportGraphDefResultsReturnOutputs(TF_ImportGraphDefResults* results, + int* num_outputs, + TF_Output** outputs) { + *num_outputs = results->return_tensors.size(); + *outputs = results->return_tensors.data(); +} + +void TF_ImportGraphDefResultsReturnOperations(TF_ImportGraphDefResults* results, + int* num_opers, + TF_Operation*** opers) { + *num_opers = results->return_nodes.size(); + *opers = results->return_nodes.data(); +} + +void TF_ImportGraphDefResultsUnusedInputMappings( + TF_ImportGraphDefResults* results, int* num_unused_input_mappings, + const char*** src_names, int** src_indexes) { + *num_unused_input_mappings = results->unused_key_names.size(); + *src_names = results->unused_key_names.data(); + *src_indexes = results->unused_key_indexes.data(); +} + +void TF_DeleteImportGraphDefResults(TF_ImportGraphDefResults* results) { + delete results; +} + static void GraphImportGraphDefLocked(TF_Graph* graph, const GraphDef& def, const TF_ImportGraphDefOptions* opts, - TF_Output* return_outputs, - int num_return_outputs, TF_Status* status) + TF_ImportGraphDefResults* tf_results, + TF_Status* status) EXCLUSIVE_LOCKS_REQUIRED(graph->mu) { - if (num_return_outputs != opts->opts.return_tensors.size()) { - status->status = InvalidArgument("Expected 'num_return_outputs' to be ", - opts->opts.return_tensors.size(), ", got ", - num_return_outputs); - return; - } - if (num_return_outputs > 0 && return_outputs == nullptr) { - status->status = InvalidArgument( - "'return_outputs' must be preallocated to length ", num_return_outputs); - return; - } const int last_node_id = graph->graph.num_node_ids(); - std::vector> return_outputs_vec; - status->status = tensorflow::ImportGraphDef( - opts->opts, def, &graph->graph, &graph->refiner, &return_outputs_vec); + tensorflow::ImportGraphDefResults results; + status->status = tensorflow::ImportGraphDef(opts->opts, def, &graph->graph, + &graph->refiner, &results); if (!status->status.ok()) return; + + // Add new nodes to name_map for (int i = last_node_id; i < graph->graph.num_node_ids(); ++i) { auto* node = graph->graph.FindNodeId(i); if (node != nullptr) graph->name_map[node->name()] = node; } - DCHECK_EQ(return_outputs_vec.size(), num_return_outputs); - for (int i = 0; i < num_return_outputs; ++i) { - return_outputs[i].oper = ToOperation(return_outputs_vec[i].first); - return_outputs[i].index = return_outputs_vec[i].second; + + // Populate return_tensors + DCHECK(tf_results->return_tensors.empty()); + tf_results->return_tensors.resize(results.return_tensors.size()); + for (int i = 0; i < results.return_tensors.size(); ++i) { + tf_results->return_tensors[i].oper = + ToOperation(results.return_tensors[i].first); + tf_results->return_tensors[i].index = results.return_tensors[i].second; + } + + // Populate return_nodes + DCHECK(tf_results->return_nodes.empty()); + tf_results->return_nodes.resize(results.return_nodes.size()); + for (int i = 0; i < results.return_nodes.size(); ++i) { + tf_results->return_nodes[i] = ToOperation(results.return_nodes[i]); + } + + // Populate unused map keys + DCHECK(tf_results->unused_key_names.empty()); + DCHECK(tf_results->unused_key_indexes.empty()); + DCHECK(tf_results->unused_key_names_data.empty()); + tf_results->unused_key_names.resize(results.unused_input_map_keys.size()); + tf_results->unused_key_indexes.resize(results.unused_input_map_keys.size()); + for (int i = 0; i < results.unused_input_map_keys.size(); ++i) { + TensorId id = results.unused_input_map_keys[i]; + tf_results->unused_key_names_data.push_back(id.first.ToString()); + tf_results->unused_key_names[i] = + tf_results->unused_key_names_data.back().c_str(); + tf_results->unused_key_indexes[i] = id.second; + } +} + +TF_ImportGraphDefResults* TF_GraphImportGraphDefWithResults( + TF_Graph* graph, const TF_Buffer* graph_def, + const TF_ImportGraphDefOptions* options, TF_Status* status) { + GraphDef def; + if (!def.ParseFromArray(graph_def->data, graph_def->length)) { + status->status = InvalidArgument("Invalid GraphDef"); + return nullptr; + } + auto results = new TF_ImportGraphDefResults(); + mutex_lock l(graph->mu); + GraphImportGraphDefLocked(graph, def, options, results, status); + if (!status->status.ok()) { + delete results; + return nullptr; } + return results; } void TF_GraphImportGraphDefWithReturnOutputs( TF_Graph* graph, const TF_Buffer* graph_def, - const TF_ImportGraphDefOptions* opts, TF_Output* return_outputs, + const TF_ImportGraphDefOptions* options, TF_Output* return_outputs, int num_return_outputs, TF_Status* status) { + if (num_return_outputs != options->opts.return_tensors.size()) { + status->status = InvalidArgument("Expected 'num_return_outputs' to be ", + options->opts.return_tensors.size(), + ", got ", num_return_outputs); + return; + } + if (num_return_outputs > 0 && return_outputs == nullptr) { + status->status = InvalidArgument( + "'return_outputs' must be preallocated to length ", num_return_outputs); + return; + } GraphDef def; if (!def.ParseFromArray(graph_def->data, graph_def->length)) { status->status = InvalidArgument("Invalid GraphDef"); return; } + TF_ImportGraphDefResults results; mutex_lock l(graph->mu); - GraphImportGraphDefLocked(graph, def, opts, return_outputs, - num_return_outputs, status); + GraphImportGraphDefLocked(graph, def, options, &results, status); + DCHECK_EQ(results.return_tensors.size(), num_return_outputs); + memcpy(return_outputs, results.return_tensors.data(), + num_return_outputs * sizeof(TF_Output)); } void TF_GraphImportGraphDef(TF_Graph* graph, const TF_Buffer* graph_def, const TF_ImportGraphDefOptions* options, TF_Status* status) { - TF_GraphImportGraphDefWithReturnOutputs(graph, graph_def, options, nullptr, 0, - status); + TF_ImportGraphDefResults* results = + TF_GraphImportGraphDefWithResults(graph, graph_def, options, status); + TF_DeleteImportGraphDefResults(results); } // While loop functions ------------------------------------------------------- @@ -1919,7 +2031,7 @@ Status CopyGraph(Graph* src_graph, Graph* dst_graph, tensorflow::ShapeRefiner* dst_refiner, const TF_Output* src_inputs, const std::vector& dst_inputs, - const tensorflow::string& prefix, + const string& prefix, const std::vector& control_deps, const TF_Output* nodes_to_return, int nreturn_nodes, std::vector* return_nodes) { @@ -1945,11 +2057,11 @@ Status CopyGraph(Graph* src_graph, Graph* dst_graph, } // TOOD(skyewm): change to OutputTensor - std::vector> return_tensors; + tensorflow::ImportGraphDefResults results; TF_RETURN_IF_ERROR( - ImportGraphDef(opts, gdef, dst_graph, dst_refiner, &return_tensors)); + ImportGraphDef(opts, gdef, dst_graph, dst_refiner, &results)); - for (const auto& pair : return_tensors) { + for (const auto& pair : results.return_tensors) { return_nodes->emplace_back(pair.first, pair.second); } return Status::OK(); @@ -2246,9 +2358,9 @@ TF_Session* TF_LoadSessionFromSavedModel( return nullptr; } - std::unordered_set tag_set; + std::unordered_set tag_set; for (int i = 0; i < tags_len; i++) { - tag_set.insert(tensorflow::string(tags[i])); + tag_set.insert(string(tags[i])); } tensorflow::SavedModelBundle bundle; @@ -2264,8 +2376,9 @@ TF_Session* TF_LoadSessionFromSavedModel( // TODO(jhseu): When Session is modified to take Graphs instead of // GraphDefs, return the Graph generated in LoadSavedModel(). TF_ImportGraphDefOptions* import_opts = TF_NewImportGraphDefOptions(); + TF_ImportGraphDefResults results; GraphImportGraphDefLocked(graph, bundle.meta_graph_def.graph_def(), - import_opts, nullptr, 0, status); + import_opts, &results, status); TF_DeleteImportGraphDefOptions(import_opts); if (TF_GetCode(status) != TF_OK) return nullptr; @@ -2361,20 +2474,20 @@ void TF_SessionRun(TF_Session* session, const TF_Buffer* run_options, TF_Run_Setup(noutputs, output_values, status); // Convert from TF_Output and TF_Tensor to a string and Tensor. - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(input_values, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = OutputName(inputs[i]); } // Convert from TF_Output to string names. - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = OutputName(outputs[i]); } // Convert from TF_Operation* to string names. - std::vector target_names(ntargets); + std::vector target_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_names[i] = target_opers[i]->node.name(); } @@ -2395,22 +2508,22 @@ void TF_SessionPRunSetup(TF_Session* session, const TF_Output* inputs, return; } - std::vector input_names(ninputs); + std::vector input_names(ninputs); for (int i = 0; i < ninputs; ++i) { input_names[i] = OutputName(inputs[i]); } - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = OutputName(outputs[i]); } - std::vector target_names(ntargets); + std::vector target_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_names[i] = target_opers[i]->node.name(); } - tensorflow::string new_handle; + string new_handle; status->status = session->session->PRunSetup(input_names, output_names, target_names, &new_handle); if (status->status.ok()) { @@ -2441,20 +2554,20 @@ void TF_SessionPRun(TF_Session* session, const char* handle, TF_Run_Setup(noutputs, output_values, status); // Convert from TF_Output and TF_Tensor to a string and Tensor. - std::vector> input_pairs(ninputs); + std::vector> input_pairs(ninputs); if (!TF_Run_Inputs(input_values, &input_pairs, status)) return; for (int i = 0; i < ninputs; ++i) { input_pairs[i].first = OutputName(inputs[i]); } // Convert from TF_Output to string names. - std::vector output_names(noutputs); + std::vector output_names(noutputs); for (int i = 0; i < noutputs; ++i) { output_names[i] = OutputName(outputs[i]); } // Convert from TF_Operation* to string names. - std::vector target_names(ntargets); + std::vector target_names(ntargets); for (int i = 0; i < ntargets; ++i) { target_names[i] = target_opers[i]->node.name(); } diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h index 76cfcd5e0d44e92126ac99075842ebdb8d5bc145..bb569d67fcbcec29e9494236abd79b3e40db91cd 100644 --- a/tensorflow/c/c_api.h +++ b/tensorflow/c/c_api.h @@ -864,6 +864,18 @@ TF_CAPI_EXPORT extern void TF_GraphToGraphDef(TF_Graph* graph, TF_Buffer* output_graph_def, TF_Status* status); +// Returns the serialized OpDef proto with name `op_name`, or a bad status if no +// such op exists. This can return OpDefs of functions copied into the graph. +TF_CAPI_EXPORT extern void TF_GraphGetOpDef(TF_Graph* graph, + const char* op_name, + TF_Buffer* output_op_def, + TF_Status* status); + +// Returns the serialized VersionDef proto for this graph. +TF_CAPI_EXPORT extern void TF_GraphVersions(TF_Graph* graph, + TF_Buffer* output_version_def, + TF_Status* status); + // TF_ImportGraphDefOptions holds options that can be passed to // TF_GraphImportGraphDef. typedef struct TF_ImportGraphDefOptions TF_ImportGraphDefOptions; @@ -907,7 +919,62 @@ TF_CAPI_EXPORT extern void TF_ImportGraphDefOptionsAddReturnOutput( TF_CAPI_EXPORT extern int TF_ImportGraphDefOptionsNumReturnOutputs( const TF_ImportGraphDefOptions* opts); +// Add an operation in `graph_def` to be returned via the `return_opers` output +// parameter of TF_GraphImportGraphDef(). +TF_CAPI_EXPORT extern void TF_ImportGraphDefOptionsAddReturnOperation( + TF_ImportGraphDefOptions* opts, const char* oper_name); + +// Returns the number of return operations added via +// TF_ImportGraphDefOptionsAddReturnOperation(). +TF_CAPI_EXPORT extern int TF_ImportGraphDefOptionsNumReturnOperations( + const TF_ImportGraphDefOptions* opts); + +// TF_ImportGraphDefResults holds results that are generated by +// TF_GraphImportGraphDefWithResults(). +typedef struct TF_ImportGraphDefResults TF_ImportGraphDefResults; + +// Fetches the return outputs requested via +// TF_ImportGraphDefOptionsAddReturnOutput(). The number of fetched outputs is +// returned in `num_outputs`. The array of return outputs is returned in +// `outputs`. `*outputs` is owned by and has the lifetime of `results`. +TF_CAPI_EXPORT extern void TF_ImportGraphDefResultsReturnOutputs( + TF_ImportGraphDefResults* results, int* num_outputs, TF_Output** outputs); + +// Fetches the return operations requested via +// TF_ImportGraphDefOptionsAddReturnOperation(). The number of fetched +// operations is returned in `num_opers`. The array of return operations is +// returned in `opers`. `*opers` is owned by and has the lifetime of `results`. +TF_CAPI_EXPORT extern void TF_ImportGraphDefResultsReturnOperations( + TF_ImportGraphDefResults* results, int* num_opers, TF_Operation*** opers); + +// Fetches any input mappings requested via +// TF_ImportGraphDefOptionsAddInputMapping() that weren't used as input to any +// node in the imported graph def. The number of fetched mappings is returned in +// `num_unused_input_mappings`. The array of each mapping's source node name is +// returned in `src_names`, and the array of each mapping's source index is +// returned in `src_indexes`. +// +// `*src_names`, `*src_indexes`, and the memory backing each string in +// `src_names` are owned by and have the lifetime of `results`. +TF_CAPI_EXPORT extern void TF_ImportGraphDefResultsUnusedInputMappings( + TF_ImportGraphDefResults* results, int* num_unused_input_mappings, + const char*** src_names, int** src_indexes); + +// Deletes a results object returned by TF_GraphImportGraphDefWithResults(). +TF_CAPI_EXPORT extern void TF_DeleteImportGraphDefResults( + TF_ImportGraphDefResults* results); + +// Import the graph serialized in `graph_def` into `graph`. Returns nullptr and +// a bad status on error. Otherwise, returns a populated +// TF_ImportGraphDefResults instance. The returned instance must be deleted via +// TF_DeleteImportGraphDefResults(). +TF_CAPI_EXPORT extern TF_ImportGraphDefResults* +TF_GraphImportGraphDefWithResults(TF_Graph* graph, const TF_Buffer* graph_def, + const TF_ImportGraphDefOptions* options, + TF_Status* status); + // Import the graph serialized in `graph_def` into `graph`. +// Convenience function for when only return outputs are needed. // // `num_return_outputs` must be the number of return outputs added (i.e. the // result of TF_ImportGraphDefOptionsNumReturnOutputs()). If @@ -919,7 +986,7 @@ TF_CAPI_EXPORT extern void TF_GraphImportGraphDefWithReturnOutputs( int num_return_outputs, TF_Status* status); // Import the graph serialized in `graph_def` into `graph`. -// Convenience function for when no return outputs have been added. +// Convenience function for when no results are needed. TF_CAPI_EXPORT extern void TF_GraphImportGraphDef( TF_Graph* graph, const TF_Buffer* graph_def, const TF_ImportGraphDefOptions* options, TF_Status* status); diff --git a/tensorflow/c/c_api_function_test.cc b/tensorflow/c/c_api_function_test.cc index 4db9a90fdc1c00d5a86de7f5f92f29a3ff4d7df9..d5580b658992413ae6f9cb79ef88751ee28ce465 100644 --- a/tensorflow/c/c_api_function_test.cc +++ b/tensorflow/c/c_api_function_test.cc @@ -1465,5 +1465,26 @@ TEST_F(CApiFunctionTest, AppendHash) { ASSERT_EQ(string("func_name_base_qaJ8jA8UmGY"), fdef.signature().name()); } +TEST_F(CApiFunctionTest, GetOpDef) { + DefineFunction(func_name_, &func_); + TF_GraphCopyFunction(host_graph_, func_, nullptr, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Test we can retrieve function OpDef from graph + TF_Buffer* buffer = TF_NewBuffer(); + TF_GraphGetOpDef(host_graph_, func_name_, buffer, s_); + ASSERT_EQ(TF_OK, TF_GetCode(s_)) << TF_Message(s_); + + // Sanity check returned OpDef + string data(static_cast(buffer->data), buffer->length); + OpDef op_def; + op_def.ParseFromString(data); + EXPECT_EQ(op_def.name(), func_name_); + EXPECT_EQ(op_def.input_arg_size(), 1); + EXPECT_EQ(op_def.output_arg_size(), 1); + + TF_DeleteBuffer(buffer); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h index 23ec1fac6f4c623464d6bc93958504a09f3f8876..bb04e01beec931a8ea66d0855eec9625d3a6a5ab 100644 --- a/tensorflow/c/c_api_internal.h +++ b/tensorflow/c/c_api_internal.h @@ -18,7 +18,9 @@ limitations under the License. #include "tensorflow/c/c_api.h" +#include #include +#include #include #include @@ -124,6 +126,20 @@ struct TF_Session { struct TF_ImportGraphDefOptions { tensorflow::ImportGraphDefOptions opts; + + // Backing memory for TensorId fields in opts. + // TODO(skyewm): it'd be better if ImportGraphDefOptions owned this. + std::list tensor_id_data; +}; + +struct TF_ImportGraphDefResults { + std::vector return_tensors; + std::vector return_nodes; + std::vector unused_key_names; + std::vector unused_key_indexes; + + // Backing memory for unused_key_names values. + std::list unused_key_names_data; }; struct TF_DeviceList { diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc index c4420290099ee10c89792210dad2604328296515..05881e619ba232de99e78f315cfa8ab9294e5137 100644 --- a/tensorflow/c/c_api_test.cc +++ b/tensorflow/c/c_api_test.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/graph/tensor_id.h" #include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -50,6 +51,11 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst); namespace { +static void ExpectHasSubstr(StringPiece s, StringPiece expected) { + EXPECT_TRUE(StringPiece(s).contains(expected)) + << "'" << s << "' does not contain '" << expected << "'"; +} + TEST(CAPI, Version) { EXPECT_STRNE("", TF_Version()); } TEST(CAPI, Status) { @@ -567,7 +573,7 @@ TEST(CAPI, ImportGraphDef) { TF_GraphToGraphDef(graph, graph_def, s); ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); - // Import it again, with a prefix, in a fresh graph. + // Import it, with a prefix, in a fresh graph. TF_DeleteGraph(graph); graph = TF_NewGraph(); TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions(); @@ -582,8 +588,8 @@ TEST(CAPI, ImportGraphDef) { ASSERT_TRUE(feed != nullptr); ASSERT_TRUE(neg != nullptr); - // Import it again, with an input mapping and return outputs, into the same - // graph. + // Import it again, with an input mapping, return outputs, and a return + // operation, into the same graph. TF_DeleteImportGraphDefOptions(opts); opts = TF_NewImportGraphDefOptions(); TF_ImportGraphDefOptionsSetPrefix(opts, "imported2"); @@ -591,9 +597,10 @@ TEST(CAPI, ImportGraphDef) { TF_ImportGraphDefOptionsAddReturnOutput(opts, "feed", 0); TF_ImportGraphDefOptionsAddReturnOutput(opts, "scalar", 0); EXPECT_EQ(2, TF_ImportGraphDefOptionsNumReturnOutputs(opts)); - TF_Output return_outputs[2]; - TF_GraphImportGraphDefWithReturnOutputs(graph, graph_def, opts, - return_outputs, 2, s); + TF_ImportGraphDefOptionsAddReturnOperation(opts, "scalar"); + EXPECT_EQ(1, TF_ImportGraphDefOptionsNumReturnOperations(opts)); + TF_ImportGraphDefResults* results = + TF_GraphImportGraphDefWithResults(graph, graph_def, opts, s); ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); TF_Operation* scalar2 = TF_GraphOperationByName(graph, "imported2/scalar"); @@ -609,11 +616,26 @@ TEST(CAPI, ImportGraphDef) { EXPECT_EQ(0, neg_input.index); // Check return outputs + TF_Output* return_outputs; + int num_return_outputs; + TF_ImportGraphDefResultsReturnOutputs(results, &num_return_outputs, + &return_outputs); + ASSERT_EQ(2, num_return_outputs); EXPECT_EQ(feed2, return_outputs[0].oper); EXPECT_EQ(0, return_outputs[0].index); EXPECT_EQ(scalar, return_outputs[1].oper); // remapped EXPECT_EQ(0, return_outputs[1].index); + // Check return operation + TF_Operation** return_opers; + int num_return_opers; + TF_ImportGraphDefResultsReturnOperations(results, &num_return_opers, + &return_opers); + ASSERT_EQ(1, num_return_opers); + EXPECT_EQ(scalar2, return_opers[0]); // not remapped + + TF_DeleteImportGraphDefResults(results); + // Import again, with control dependencies, into the same graph. TF_DeleteImportGraphDefOptions(opts); opts = TF_NewImportGraphDefOptions(); @@ -683,6 +705,113 @@ TEST(CAPI, ImportGraphDef) { TF_DeleteStatus(s); } +TEST(CAPI, ImportGraphDef_WithReturnOutputs) { + TF_Status* s = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + + // Create a graph with two nodes: x and 3 + Placeholder(graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "feed") != nullptr); + TF_Operation* oper = ScalarConst(3, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "scalar") != nullptr); + Neg(oper, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "neg") != nullptr); + + // Export to a GraphDef. + TF_Buffer* graph_def = TF_NewBuffer(); + TF_GraphToGraphDef(graph, graph_def, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + // Import it in a fresh graph with return outputs. + TF_DeleteGraph(graph); + graph = TF_NewGraph(); + TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions(); + TF_ImportGraphDefOptionsAddReturnOutput(opts, "feed", 0); + TF_ImportGraphDefOptionsAddReturnOutput(opts, "scalar", 0); + EXPECT_EQ(2, TF_ImportGraphDefOptionsNumReturnOutputs(opts)); + TF_Output return_outputs[2]; + TF_GraphImportGraphDefWithReturnOutputs(graph, graph_def, opts, + return_outputs, 2, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + TF_Operation* scalar = TF_GraphOperationByName(graph, "scalar"); + TF_Operation* feed = TF_GraphOperationByName(graph, "feed"); + TF_Operation* neg = TF_GraphOperationByName(graph, "neg"); + ASSERT_TRUE(scalar != nullptr); + ASSERT_TRUE(feed != nullptr); + ASSERT_TRUE(neg != nullptr); + + // Check return outputs + EXPECT_EQ(feed, return_outputs[0].oper); + EXPECT_EQ(0, return_outputs[0].index); + EXPECT_EQ(scalar, return_outputs[1].oper); + EXPECT_EQ(0, return_outputs[1].index); + + TF_DeleteImportGraphDefOptions(opts); + TF_DeleteBuffer(graph_def); + TF_DeleteGraph(graph); + TF_DeleteStatus(s); +} + +TEST(CAPI, ImportGraphDef_UnusedInputMappings) { + TF_Status* s = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + + // Create a graph with two nodes: x and 3 + Placeholder(graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "feed") != nullptr); + TF_Operation* oper = ScalarConst(3, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "scalar") != nullptr); + Neg(oper, graph, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + ASSERT_TRUE(TF_GraphOperationByName(graph, "neg") != nullptr); + + // Export to a GraphDef. + TF_Buffer* graph_def = TF_NewBuffer(); + TF_GraphToGraphDef(graph, graph_def, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + // Import it in a fresh graph. + TF_DeleteGraph(graph); + graph = TF_NewGraph(); + TF_ImportGraphDefOptions* opts = TF_NewImportGraphDefOptions(); + TF_GraphImportGraphDef(graph, graph_def, opts, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + TF_Operation* scalar = TF_GraphOperationByName(graph, "scalar"); + + // Import it in a fresh graph with an unused input mapping. + TF_DeleteImportGraphDefOptions(opts); + opts = TF_NewImportGraphDefOptions(); + TF_ImportGraphDefOptionsSetPrefix(opts, "imported"); + TF_ImportGraphDefOptionsAddInputMapping(opts, "scalar", 0, {scalar, 0}); + TF_ImportGraphDefOptionsAddInputMapping(opts, "fake", 0, {scalar, 0}); + TF_ImportGraphDefResults* results = + TF_GraphImportGraphDefWithResults(graph, graph_def, opts, s); + ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s); + + // Check unused input mappings + int num_unused_input_mappings; + const char** src_names; + int* src_indexes; + TF_ImportGraphDefResultsUnusedInputMappings( + results, &num_unused_input_mappings, &src_names, &src_indexes); + ASSERT_EQ(1, num_unused_input_mappings); + EXPECT_EQ(string("fake"), string(src_names[0])); + EXPECT_EQ(0, src_indexes[0]); + + TF_DeleteImportGraphDefResults(results); + TF_DeleteImportGraphDefOptions(opts); + TF_DeleteBuffer(graph_def); + TF_DeleteGraph(graph); + TF_DeleteStatus(s); +} + TEST(CAPI, Session) { TF_Status* s = TF_NewStatus(); TF_Graph* graph = TF_NewGraph(); @@ -837,6 +966,31 @@ TEST(CAPI, ShapeInferenceError) { TF_DeleteStatus(status); } +TEST(CAPI, GetOpDef) { + TF_Status* status = TF_NewStatus(); + TF_Graph* graph = TF_NewGraph(); + TF_Buffer* buffer = TF_NewBuffer(); + + TF_GraphGetOpDef(graph, "Add", buffer, status); + ASSERT_EQ(TF_OK, TF_GetCode(status)); + const OpDef* expected_op_def; + TF_ASSERT_OK(OpRegistry::Global()->LookUpOpDef("Add", &expected_op_def)); + string expected_serialized; + expected_op_def->SerializeToString(&expected_serialized); + string actual_string(reinterpret_cast(buffer->data), + buffer->length); + EXPECT_EQ(expected_serialized, actual_string); + + TF_GraphGetOpDef(graph, "MyFakeOp", buffer, status); + EXPECT_EQ(TF_NOT_FOUND, TF_GetCode(status)); + ExpectHasSubstr(TF_Message(status), + "Op type not registered 'MyFakeOp' in binary"); + + TF_DeleteBuffer(buffer); + TF_DeleteGraph(graph); + TF_DeleteStatus(status); +} + void StringVectorToArrays(const std::vector& v, std::unique_ptr* ptrs, std::unique_ptr* lens) { diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index 96f3c3e195e7025252c1e3cda5436237ad89257b..c77896b80b478cd34d3502e1061a7e76204ba021 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -3,6 +3,7 @@ licenses(["notice"]) # Apache 2.0 load( "//tensorflow:tensorflow.bzl", + "tf_cuda_cc_test", "tf_cc_test", "tf_copts", "tf_cuda_library", @@ -50,7 +51,7 @@ tf_cuda_library( ], ) -tf_cc_test( +tf_cuda_cc_test( name = "c_api_test", srcs = ["c_api_test.cc"], deps = [ diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 514a4010bc81bb280c3a1208b57a5db752f52f8a..8359de62b7ff690fec9f6a0e3280f947c62f8b6e 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -54,9 +54,23 @@ string DeviceName(tensorflow::Device* d) { extern "C" { -TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) { +TFE_ContextOptions* TFE_NewContextOptions() { return new TFE_ContextOptions; } + +void TFE_ContextOptionsSetConfig(TFE_ContextOptions* options, const void* proto, + size_t proto_len, TF_Status* status) { + TF_SetConfig(&options->session_options, proto, proto_len, status); +} + +void TFE_ContextOptionsSetDevicePlacementPolicy( + TFE_ContextOptions* options, TFE_ContextDevicePlacementPolicy policy) { + options->policy = policy; +} + +void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; } + +TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { TF_Graph* graph = TF_NewGraph(); - TF_Session* session = TF_NewSession(graph, opts, status); + TF_Session* session = TF_NewSession(graph, &opts->session_options, status); if (status->status.ok()) { if (session->device_mgr == nullptr || session->devices.empty()) { status->status = tensorflow::errors::InvalidArgument( @@ -71,9 +85,10 @@ TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, TF_Status* status) { } TFE_Context* ret = new TFE_Context(session); + ret->policy = opts->policy; ret->pflr.reset(new tensorflow::ProcessFunctionLibraryRuntime( - ret->session->device_mgr, opts->options.env, TF_GRAPH_DEF_VERSION, - &ret->func_lib_def, {})); + ret->session->device_mgr, opts->session_options.options.env, + TF_GRAPH_DEF_VERSION, &ret->func_lib_def, {})); ret->rendezvous = new tensorflow::IntraProcessRendezvous(ret->session->device_mgr); @@ -408,8 +423,10 @@ void TFE_OpSetAttrShapeList(TFE_Op* op, const char* attr_name, namespace { tensorflow::Status ValidateInputTypeAndPlacement( - tensorflow::Device* host_device, tensorflow::Device* op_device, TFE_Op* op, - const tensorflow::OpKernel* kernel) { + TFE_Context* ctx, tensorflow::Device* host_device, + tensorflow::Device* op_device, TFE_Op* op, + const tensorflow::OpKernel* kernel, + std::vector* copied_tensors) { const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types(); if (memtypes.size() != op->inputs.size()) { return tensorflow::errors::InvalidArgument( @@ -421,11 +438,50 @@ tensorflow::Status ValidateInputTypeAndPlacement( const tensorflow::Device* actual_device = op->input_devices[i] == nullptr ? host_device : op->input_devices[i]; if (expected_device != actual_device) { - return tensorflow::errors::InvalidArgument( - "cannot compute ", op->name, " as input #", i, - " was expected to be on ", expected_device->name(), - " but is actually on ", actual_device->name(), - " (operation running on ", op_device->name(), ")"); + switch (ctx->policy) { + case TFE_DEVICE_PLACEMENT_EXPLICIT: + // TODO(xpan): See if we could bubble python related error up + // to python level. + return tensorflow::errors::InvalidArgument( + "Tensors on conflicting devices:" + " cannot compute ", + op->name, " as input #", i, " was expected to be on ", + expected_device->name(), " but is actually on ", + actual_device->name(), " (operation running on ", + op_device->name(), ")", + " Tensors can be copied explicitly using .gpu() or .cpu()," + " or transparently copied by using tfe.enable_eager_execution(" + "tfe.DEVICE_PLACEMENT_SILENT). Copying tensors between devices" + " may slow down your model"); + case TFE_DEVICE_PLACEMENT_WARN: + LOG(WARNING) << "before computing " << op->name << " input #" << i + << " was expected to be on " << expected_device->name() + << " but is actually on " << actual_device->name() + << " (operation running on " << op_device->name() + << "). This triggers a copy which can be a performance " + "bottleneck."; + break; + case TFE_DEVICE_PLACEMENT_SILENT: // Do nothing. + break; + } + // We are only here if the policy is warn or silent copies, so we should + // trigger a copy. + TFE_TensorHandle original{op->inputs[i], op->input_devices[i]}; + TF_Status* s = TF_NewStatus(); + TFE_TensorHandle* copied_tensor = TFE_TensorHandleCopyToDevice( + &original, ctx, expected_device->name().c_str(), s); + if (!s->status.ok()) { + tensorflow::Status status = s->status; + delete s; + return tensorflow::errors::Internal( + "Failed copying input tensor from ", actual_device->name(), " to ", + expected_device->name(), " in order to run ", op->name, ": ", + status.error_message()); + } + op->inputs[i] = copied_tensor->t; + copied_tensors->push_back(copied_tensor); + op->input_devices[i] = copied_tensor->d; + delete s; } if (op->inputs[i].dtype() != kernel->input_type(i)) { return tensorflow::errors::InvalidArgument( @@ -468,10 +524,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, } tensorflow::gtl::InsertOrUpdate(&(ctx->kernel_cache), cache_key, kernel); } - status->status = ValidateInputTypeAndPlacement(ctx->devices()[0], device, op, - kernel->kernel()); + std::vector copied_tensors; + status->status = ValidateInputTypeAndPlacement( + ctx, ctx->devices()[0], device, op, kernel->kernel(), &copied_tensors); output_memory_types = &kernel->kernel()->output_memory_types(); if (!status->status.ok()) { + for (auto* t : copied_tensors) { + TFE_DeleteTensorHandle(t); + } return; } // WARNING: kernel->Run utilizes the FunctionLibraryRuntime @@ -483,6 +543,9 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals, // sense for FunctionLibraryRuntime to ensure thread-safe access to // FunctionLibraryDefinition?). status->status = kernel->Run(&op->inputs, &outputs); + for (auto* t : copied_tensors) { + TFE_DeleteTensorHandle(t); + } if (!status->status.ok()) return; *num_retvals = std::min(*num_retvals, outputs.size()); for (int i = 0; i < *num_retvals; ++i) { diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h index 9bfa63711b5360b33819434f9a551030e0f988c8..865580c5f3a823d9cf49fe460bd007e3b3b88767 100644 --- a/tensorflow/c/eager/c_api.h +++ b/tensorflow/c/eager/c_api.h @@ -43,14 +43,46 @@ limitations under the License. extern "C" { #endif +typedef struct TFE_ContextOptions TFE_ContextOptions; + +// Return a new options object. +TF_CAPI_EXPORT extern TFE_ContextOptions* TFE_NewContextOptions(); + +// Set the config in TF_ContextOptions.options. +// config should be a serialized tensorflow.ConfigProto proto. +// If config was not parsed successfully as a ConfigProto, record the +// error information in *status. +TF_CAPI_EXPORT extern void TFE_ContextOptionsSetConfig( + TFE_ContextOptions* options, const void* proto, size_t proto_len, + TF_Status* status); + +// Controls how to act when we try to run an operation on a given device but +// some input tensors are not on that device. +typedef enum TFE_ContextDevicePlacementPolicy { + // The default: running operations with input tensors on the wrong device will + // fail. + TFE_DEVICE_PLACEMENT_EXPLICIT = 0, + // Copy the tensor to the right device but log a warning. + TFE_DEVICE_PLACEMENT_WARN = 1, + // Silently copy the tensor, which has a performance cost since the + // operation will be blocked till the copy completes. + TFE_DEVICE_PLACEMENT_SILENT = 2, +} TFE_ContextDevicePlacementPolicy; + +TF_CAPI_EXPORT extern void TFE_ContextOptionsSetDevicePlacementPolicy( + TFE_ContextOptions*, TFE_ContextDevicePlacementPolicy); + +// Destroy an options object. +TF_CAPI_EXPORT extern void TFE_DeleteContextOptions(TFE_ContextOptions*); + // "Context" under which operations/functions are executed. It encapsulates // things like the available devices, resource manager etc. // // TODO(ashankar): Merge with TF_Session? typedef struct TFE_Context TFE_Context; -TF_CAPI_EXPORT extern TFE_Context* TFE_NewContext(const TF_SessionOptions* opts, - TF_Status* status); +TF_CAPI_EXPORT extern TFE_Context* TFE_NewContext( + const TFE_ContextOptions* opts, TF_Status* status); TF_CAPI_EXPORT extern void TFE_DeleteContext(TFE_Context* ctx, TF_Status* status); TF_CAPI_EXPORT extern TF_DeviceList* TFE_ContextListDevices(TFE_Context* ctx, TF_Status* status); diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h index 712526f17002a612a145f80538977fedfde00038..0971e2ab2fe98cc8bf6f631f41d5adce90ee7051 100644 --- a/tensorflow/c/eager/c_api_internal.h +++ b/tensorflow/c/eager/c_api_internal.h @@ -35,9 +35,16 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/thread_annotations.h" +struct TFE_ContextOptions { + TF_SessionOptions session_options; + TFE_ContextDevicePlacementPolicy policy{TFE_DEVICE_PLACEMENT_EXPLICIT}; +}; + struct TFE_Context { explicit TFE_Context(TF_Session* s) : session(s) {} + TFE_ContextDevicePlacementPolicy policy; + // TFE_Context is an extension of TF_Session. And TF_Session needs a TF_Graph. TF_Session* session; tensorflow::Rendezvous* rendezvous; diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc index 72e0fe8a1565a9a717c01aed83044cab2dd2dfbc..4af91b8853d0e85570bad136752a9d0a04b87da5 100644 --- a/tensorflow/c/eager/c_api_test.cc +++ b/tensorflow/c/eager/c_api_test.cc @@ -62,10 +62,10 @@ TFE_Op* MatMulOp(TFE_Context* ctx, TFE_TensorHandle* a, TFE_TensorHandle* b) { void BM_InitOp(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* m = TestMatrixTensorHandle(); tensorflow::testing::StartTiming(); @@ -84,10 +84,10 @@ BENCHMARK(BM_InitOp); void BM_Execute(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* m = TestMatrixTensorHandle(); TFE_Op* matmul = MatMulOp(ctx, m, m); @@ -109,9 +109,9 @@ BENCHMARK(BM_Execute); TEST(CAPI, Context) { TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TF_DeviceList* devices = TFE_ContextListDevices(ctx, status); EXPECT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); @@ -150,9 +150,9 @@ TEST(CAPI, TensorHandle) { TEST(CAPI, TensorHandleCopyBetweenDevices) { std::unique_ptr status( TF_NewStatus(), TF_DeleteStatus); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status.get()); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); TFE_TensorHandle* hcpu = TestMatrixTensorHandle(); @@ -216,12 +216,58 @@ TEST(CAPI, TensorHandleCopyBetweenDevices) { EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); } +TEST(CAPI, TensorHandleSilentCopy) { + std::unique_ptr status( + TF_NewStatus(), TF_DeleteStatus); + TFE_ContextOptions* opts = TFE_NewContextOptions(); + TFE_ContextOptionsSetDevicePlacementPolicy(opts, TFE_DEVICE_PLACEMENT_SILENT); + TFE_Context* ctx = TFE_NewContext(opts, status.get()); + TFE_DeleteContextOptions(opts); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + + TFE_TensorHandle* hcpu = TestMatrixTensorHandle(); + TF_Tensor* t = TFE_TensorHandleResolve(hcpu, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + + TF_DeviceList* devices = TFE_ContextListDevices(ctx, status.get()); + ASSERT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); + const int num_devices = TF_DeviceListCount(devices); + + // Disable the test if no GPU is present. + if (num_devices > 1) { + const int device_to_use = 1; + const string name(TF_DeviceListName(devices, device_to_use, status.get())); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + + TFE_TensorHandle* hgpu = + TFE_TensorHandleCopyToDevice(hcpu, ctx, name.c_str(), status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + + TFE_Op* matmul = MatMulOp(ctx, hcpu, hgpu); + TFE_OpSetDevice(matmul, name.c_str(), status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + TFE_TensorHandle* retvals[1]; + int num_retvals = 1; + TFE_Execute(matmul, &retvals[0], &num_retvals, status.get()); + ASSERT_TRUE(TF_GetCode(status.get()) == TF_OK) << TF_Message(status.get()); + TFE_DeleteOp(matmul); + TFE_DeleteTensorHandle(retvals[0]); + TFE_DeleteTensorHandle(hgpu); + } + + TF_DeleteDeviceList(devices); + TF_DeleteTensor(t); + TFE_DeleteTensorHandle(hcpu); + TFE_DeleteContext(ctx, status.get()); + EXPECT_EQ(TF_OK, TF_GetCode(status.get())) << TF_Message(status.get()); +} + TEST(CAPI, Execute) { TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* m = TestMatrixTensorHandle(); TFE_Op* matmul = MatMulOp(ctx, m, m); @@ -285,10 +331,10 @@ string MatMulFunction() { TEST(CAPI, FunctionDefAndExecute) { TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); string function_def = MatMulFunction(); TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), @@ -326,10 +372,10 @@ TEST(CAPI, FunctionDefAndExecute) { void BM_ExecuteFunction(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); string function_def = MatMulFunction(); TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), @@ -406,10 +452,10 @@ TEST(CAPI, Variables) { // Variables use resource handles, so this is really a test for resource // tensor handling. TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* var_handle = CreateVariable(ctx, 12.0, status); ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); @@ -446,10 +492,10 @@ TEST(CAPI, Variables) { void BM_ReadVariable(int iters) { tensorflow::testing::StopTiming(); TF_Status* status = TF_NewStatus(); - TF_SessionOptions* opts = TF_NewSessionOptions(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); TFE_Context* ctx = TFE_NewContext(opts, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); - TF_DeleteSessionOptions(opts); + TFE_DeleteContextOptions(opts); TFE_TensorHandle* var_handle = CreateVariable(ctx, 5.0, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status); diff --git a/tensorflow/c/while_loop_test.cc b/tensorflow/c/while_loop_test.cc index 2423d83dda93938aa1a2ba0ed0ed7356bd65d39f..d2d887f32c44af5980b50785f282187d0f6fcff4 100644 --- a/tensorflow/c/while_loop_test.cc +++ b/tensorflow/c/while_loop_test.cc @@ -318,7 +318,7 @@ TEST_F(CApiWhileLoopTest, InvalidCondOutputNode) { // TODO(skyewm): this error message could be more informative. Add explicit // checks for this case in the while loop implementation? ExpectError(TF_INVALID_ARGUMENT, - "Requested return node 'p0' not found in graph def"); + "Requested return tensor 'p0:0' not found in graph def"); } TEST_F(CApiWhileLoopTest, InvalidCondOutputIndex) { @@ -358,7 +358,7 @@ TEST_F(CApiWhileLoopTest, InvalidBodyOutputNode) { // TODO(skyewm): this error message could be more informative. Add explicit // checks for this case in the while loop implementation? ExpectError(TF_INVALID_ARGUMENT, - "Requested return node 'p0' not found in graph def"); + "Requested return tensor 'p0:0' not found in graph def"); } // TODO(skyewm): enable this when it works (currently segfaults!) @@ -389,7 +389,7 @@ TEST_F(CApiWhileLoopTest, WrongGraph) { params_->body_outputs[0] = inputs_[0]; // TODO(skyewm): improve error message ExpectError(TF_INVALID_ARGUMENT, - "Requested return node 'p0' not found in graph def"); + "Requested return tensor 'p0:0' not found in graph def"); } TEST_F(CApiWhileLoopTest, BadTypes) { diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD index 67b2e4b81a985731ad5e41ce68a5aeaa9fcef6b9..d29ad3ebcbe29087d5572b51c7713e0c98d0d840 100644 --- a/tensorflow/cc/saved_model/BUILD +++ b/tensorflow/cc/saved_model/BUILD @@ -89,7 +89,6 @@ tf_cc_test( filegroup( name = "saved_model_half_plus_two", srcs = glob([ - "testdata/half_plus_two_forward_compatibility/**", "testdata/half_plus_two_pbtxt/**", "testdata/half_plus_two_main_op/**", "testdata/half_plus_two/**", diff --git a/tensorflow/cc/saved_model/loader.cc b/tensorflow/cc/saved_model/loader.cc index 462308a48f1e64d368b2a29cde8b6180b2552f2f..f98abc8a817eca7bc129bb03a2ad31b97d957065 100644 --- a/tensorflow/cc/saved_model/loader.cc +++ b/tensorflow/cc/saved_model/loader.cc @@ -18,7 +18,6 @@ limitations under the License. #include #include "tensorflow/cc/saved_model/constants.h" -#include "tensorflow/core/framework/graph_def_util.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/monitoring/counter.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -225,18 +224,6 @@ Status GetAssetFileDefs(const MetaGraphDef& meta_graph_def, return Status::OK(); } -// For forward compatibility, remove new default attributes from the graph def -// that were not present in the consumer (e.g. If graph was exported using -// code that's newer than the server and a new default attr was added). -Status RemoveNewDefaultAttrsFromMetaGraphDef(MetaGraphDef* meta_graph_def) { - OpListOpRegistry producer_op_registry( - &meta_graph_def->meta_info_def().stripped_op_list()); - OpRegistry* consumer_op_registry = OpRegistry::Global(); - return RemoveNewDefaultAttrsFromGraphDef(meta_graph_def->mutable_graph_def(), - *consumer_op_registry, - producer_op_registry, nullptr); -} - Status LoadSavedModelInternal(const SessionOptions& session_options, const RunOptions& run_options, const string& export_dir, @@ -254,9 +241,6 @@ Status LoadSavedModelInternal(const SessionOptions& session_options, TF_RETURN_IF_ERROR( FindMetaGraphDefToLoad(saved_model_proto, tags, &bundle->meta_graph_def)); - TF_RETURN_IF_ERROR( - RemoveNewDefaultAttrsFromMetaGraphDef(&bundle->meta_graph_def)); - TF_RETURN_IF_ERROR(LoadMetaGraphIntoSession( bundle->meta_graph_def, session_options, &bundle->session)); diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc index 6dd14837b5e9f31395a26b49082e3339817473d0..0ad6b33bba5fcceaca68e2f179cef2232c689a80 100644 --- a/tensorflow/cc/saved_model/loader_test.cc +++ b/tensorflow/cc/saved_model/loader_test.cc @@ -29,12 +29,10 @@ limitations under the License. namespace tensorflow { namespace { -constexpr char kTestDataForwardCompatibility[] = - "cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123"; -constexpr char kTestDataMainOp[] = - "cc/saved_model/testdata/half_plus_two_main_op/00000123"; constexpr char kTestDataPbTxt[] = "cc/saved_model/testdata/half_plus_two_pbtxt/00000123"; +constexpr char kTestDataMainOp[] = + "cc/saved_model/testdata/half_plus_two_main_op/00000123"; constexpr char kTestDataSharded[] = "cc/saved_model/testdata/half_plus_two/00000123"; @@ -169,24 +167,6 @@ TEST_F(LoaderTest, PbtxtFormat) { CheckSavedModelBundle(export_dir, bundle); } -// Forward compatibility graph has a new attr with a default value equal to the -// value used by the server. If we handle new default attrs correctly, this test -// will pass. This simulates adding new atts to the training code while server -// code lags behind. -TEST_F(LoaderTest, ForwardCompatibility) { - SavedModelBundle bundle; - SessionOptions session_options; - RunOptions run_options; - - // TODO(b/67753689): Add support for regenerating this model in the export - // code. - const string export_dir = - io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataForwardCompatibility); - TF_ASSERT_OK(LoadSavedModel(session_options, run_options, export_dir, - {kSavedModelTagServe}, &bundle)); - CheckSavedModelBundle(export_dir, bundle); -} - TEST_F(LoaderTest, MainOpFormat) { SavedModelBundle bundle; SessionOptions session_options; diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt deleted file mode 100644 index f9ff036688007836524129e23f5cf82edd1e8910..0000000000000000000000000000000000000000 --- a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/assets/foo.txt +++ /dev/null @@ -1 +0,0 @@ -asset-file-contents \ No newline at end of file diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt deleted file mode 100755 index e799b3579c6e79de83989d4f19662becae4a5301..0000000000000000000000000000000000000000 --- a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/saved_model.pbtxt +++ /dev/null @@ -1,2728 +0,0 @@ -saved_model_schema_version: 1 -meta_graphs { - meta_info_def { - stripped_op_list { - op { - name: "Add" - input_arg { - name: "x" - type_attr: "T" - } - input_arg { - name: "y" - type_attr: "T" - } - output_arg { - name: "z" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_FLOAT - type: DT_DOUBLE - type: DT_UINT8 - type: DT_INT8 - type: DT_INT16 - type: DT_INT32 - type: DT_INT64 - type: DT_COMPLEX64 - type: DT_COMPLEX128 - type: DT_STRING - } - } - } - } - op { - name: "Assign" - input_arg { - name: "ref" - type_attr: "T" - is_ref: true - } - input_arg { - name: "value" - type_attr: "T" - } - output_arg { - name: "output_ref" - type_attr: "T" - is_ref: true - } - attr { - name: "T" - type: "type" - } - attr { - name: "validate_shape" - type: "bool" - default_value { - b: true - } - } - attr { - name: "use_locking" - type: "bool" - default_value { - b: true - } - } - allows_uninitialized_input: true - } - op { - name: "Const" - output_arg { - name: "output" - type_attr: "dtype" - } - attr { - name: "value" - type: "tensor" - } - attr { - name: "dtype" - type: "type" - } - } - op { - name: "Identity" - input_arg { - name: "input" - type_attr: "T" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - } - } - op { - name: "MergeV2Checkpoints" - input_arg { - name: "checkpoint_prefixes" - type: DT_STRING - } - input_arg { - name: "destination_prefix" - type: DT_STRING - } - attr { - name: "delete_old_dirs" - type: "bool" - default_value { - b: true - } - } - } - op { - name: "Mul" - input_arg { - name: "x" - type_attr: "T" - } - input_arg { - name: "y" - type_attr: "T" - } - output_arg { - name: "z" - type_attr: "T" - } - attr { - name: "T" - type: "type" - allowed_values { - list { - type: DT_HALF - type: DT_FLOAT - type: DT_DOUBLE - type: DT_UINT8 - type: DT_INT8 - type: DT_UINT16 - type: DT_INT16 - type: DT_INT32 - type: DT_INT64 - type: DT_COMPLEX64 - type: DT_COMPLEX128 - } - } - } - is_commutative: true - } - op { - name: "NoOp" - } - op { - name: "Pack" - input_arg { - name: "values" - type_attr: "T" - number_attr: "N" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "N" - type: "int" - has_minimum: true - minimum: 1 - } - attr { - name: "T" - type: "type" - } - attr { - name: "axis" - type: "int" - default_value { - i: 0 - } - } - } - op { - name: "ParseExample" - input_arg { - name: "serialized" - type_attr: "TInputs" - } - input_arg { - name: "names" - type: DT_STRING - } - input_arg { - name: "sparse_keys" - type: DT_STRING - number_attr: "Nsparse" - } - input_arg { - name: "dense_keys" - type: DT_STRING - number_attr: "Ndense" - } - input_arg { - name: "dense_defaults" - type_list_attr: "Tdense" - } - output_arg { - name: "sparse_indices" - type: DT_INT64 - number_attr: "Nsparse" - } - output_arg { - name: "sparse_values" - type_list_attr: "sparse_types" - } - output_arg { - name: "sparse_shapes" - type: DT_INT64 - number_attr: "Nsparse" - } - output_arg { - name: "dense_values" - type_list_attr: "Tdense" - } - attr { - name: "Nsparse" - type: "int" - has_minimum: true - } - attr { - name: "TInputs" - type: "type" - default_value { - type: DT_STRING - } - allowed_values { - list { - type: DT_STRING - type: DT_INT64 - } - } - } - attr { - name: "Ndense" - type: "int" - has_minimum: true - } - attr { - name: "sparse_types" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "Tdense" - type: "list(type)" - has_minimum: true - allowed_values { - list { - type: DT_FLOAT - type: DT_INT64 - type: DT_STRING - } - } - } - attr { - name: "dense_shapes" - type: "list(shape)" - has_minimum: true - } - } - op { - name: "Placeholder" - output_arg { - name: "output" - type_attr: "dtype" - } - attr { - name: "dtype" - type: "type" - } - attr { - name: "shape" - type: "shape" - default_value { - shape { - unknown_rank: true - } - } - } - } - op { - name: "Reshape" - input_arg { - name: "tensor" - type_attr: "T" - } - input_arg { - name: "shape" - type_attr: "Tshape" - } - output_arg { - name: "output" - type_attr: "T" - } - attr { - name: "T" - type: "type" - } - attr { - name: "Tshape" - type: "type" - default_value { - type: DT_INT32 - } - allowed_values { - list { - type: DT_INT32 - type: DT_INT64 - } - } - } - } - op { - name: "RestoreV2" - input_arg { - name: "prefix" - type: DT_STRING - } - input_arg { - name: "tensor_names" - type: DT_STRING - } - input_arg { - name: "shape_and_slices" - type: DT_STRING - } - output_arg { - name: "tensors" - type_list_attr: "dtypes" - } - attr { - name: "dtypes" - type: "list(type)" - has_minimum: true - minimum: 1 - } - } - op { - name: "SaveV2" - input_arg { - name: "prefix" - type: DT_STRING - } - input_arg { - name: "tensor_names" - type: DT_STRING - } - input_arg { - name: "shape_and_slices" - type: DT_STRING - } - input_arg { - name: "tensors" - type_list_attr: "dtypes" - } - attr { - name: "dtypes" - type: "list(type)" - has_minimum: true - minimum: 1 - } - } - op { - name: "ShardedFilename" - input_arg { - name: "basename" - type: DT_STRING - } - input_arg { - name: "shard" - type: DT_INT32 - } - input_arg { - name: "num_shards" - type: DT_INT32 - } - output_arg { - name: "filename" - type: DT_STRING - } - } - op { - name: "StringJoin" - input_arg { - name: "inputs" - type: DT_STRING - number_attr: "N" - } - output_arg { - name: "output" - type: DT_STRING - } - attr { - name: "N" - type: "int" - has_minimum: true - minimum: 1 - } - attr { - name: "separator" - type: "string" - default_value { - s: "" - } - } - } - op { - name: "VariableV2" - output_arg { - name: "ref" - type_attr: "dtype" - is_ref: true - } - attr { - name: "shape" - type: "shape" - } - attr { - name: "dtype" - type: "type" - } - attr { - name: "container" - type: "string" - default_value { - s: "" - } - } - attr { - name: "shared_name" - type: "string" - default_value { - s: "" - } - } - is_stateful: true - } - } - tags: "serve" - tensorflow_version: "1.1.0-rc2" - tensorflow_git_version: "unknown" - } - graph_def { - node { - name: "a/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - } - float_val: 0.5 - } - } - } - } - node { - name: "a" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "a/Assign" - op: "Assign" - input: "a" - input: "a/initial_value" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@a" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "a/read" - op: "Identity" - input: "a" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@a" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "b/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - } - float_val: 2.0 - } - } - } - } - node { - name: "b" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "b/Assign" - op: "Assign" - input: "b" - input: "b/initial_value" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@b" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "b/read" - op: "Identity" - input: "b" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@b" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "c/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - } - float_val: 3.0 - } - } - } - } - node { - name: "c" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "c/Assign" - op: "Assign" - input: "c" - input: "c/initial_value" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@c" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "c/read" - op: "Identity" - input: "c" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@c" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "tf_example" - op: "Placeholder" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "shape" - value { - shape { - unknown_rank: true - } - } - } - } - node { - name: "ParseExample/Const" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - dim { - } - } - } - } - } - } - node { - name: "ParseExample/key_x2" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_FLOAT - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_FLOAT - tensor_shape { - dim { - size: 1 - } - } - float_val: 0.0 - } - } - } - } - node { - name: "ParseExample/Reshape/shape" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - dim { - size: 1 - } - } - int_val: 1 - } - } - } - } - node { - name: "ParseExample/Reshape" - op: "Reshape" - input: "ParseExample/key_x2" - input: "ParseExample/Reshape/shape" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "Tshape" - value { - type: DT_INT32 - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - } - node { - name: "ParseExample/ParseExample/names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - } - } - } - } - } - } - node { - name: "ParseExample/ParseExample/dense_keys_0" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "x" - } - } - } - } - node { - name: "ParseExample/ParseExample/dense_keys_1" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "x2" - } - } - } - } - node { - name: "ParseExample/ParseExample" - op: "ParseExample" - input: "tf_example" - input: "ParseExample/ParseExample/names" - input: "ParseExample/ParseExample/dense_keys_0" - input: "ParseExample/ParseExample/dense_keys_1" - input: "ParseExample/Const" - input: "ParseExample/Reshape" - attr { - key: "Ndense" - value { - i: 2 - } - } - attr { - key: "TInputs" - value { - type: DT_STRING - } - } - attr { - key: "Nsparse" - value { - i: 0 - } - } - attr { - key: "Tdense" - value { - list { - type: DT_FLOAT - type: DT_FLOAT - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - attr { - key: "dense_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "sparse_types" - value { - list { - } - } - } - } - node { - name: "x" - op: "Identity" - input: "ParseExample/ParseExample" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Mul" - op: "Mul" - input: "a/read" - input: "x" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "y" - op: "Add" - input: "Mul" - input: "b/read" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Mul_1" - op: "Mul" - input: "a/read" - input: "x" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "y2" - op: "Add" - input: "Mul_1" - input: "c/read" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "x2" - op: "Identity" - input: "ParseExample/ParseExample:1" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Mul_2" - op: "Mul" - input: "a/read" - input: "x2" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "y3" - op: "Add" - input: "Mul_2" - input: "c/read" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - } - } - node { - name: "Const" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "/tmp/original/export/assets/foo.txt" - } - } - } - } - node { - name: "filename_tensor/initial_value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "foo.txt" - } - } - } - } - node { - name: "filename_tensor" - op: "VariableV2" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "container" - value { - s: "" - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "shape" - value { - shape { - } - } - } - attr { - key: "shared_name" - value { - s: "" - } - } - } - node { - name: "filename_tensor/Assign" - op: "Assign" - input: "filename_tensor" - input: "filename_tensor/initial_value" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@filename_tensor" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "filename_tensor/read" - op: "Identity" - input: "filename_tensor" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@filename_tensor" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "Assign/value" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "foo.txt" - } - } - } - } - node { - name: "Assign" - op: "Assign" - input: "filename_tensor" - input: "Assign/value" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@filename_tensor" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: false - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "init" - op: "NoOp" - input: "^a/Assign" - input: "^b/Assign" - input: "^c/Assign" - } - node { - name: "group_deps" - op: "NoOp" - input: "^Assign" - } - node { - name: "save/Const" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "model" - } - } - } - } - node { - name: "save/StringJoin/inputs_1" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - } - string_val: "_temp_80e928f1e0c844239d136d1ea966099d/part" - } - } - } - } - node { - name: "save/StringJoin" - op: "StringJoin" - input: "save/Const" - input: "save/StringJoin/inputs_1" - attr { - key: "N" - value { - i: 2 - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "separator" - value { - s: "" - } - } - } - node { - name: "save/num_shards" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - } - int_val: 1 - } - } - } - } - node { - name: "save/ShardedFilename/shard" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "dtype" - value { - type: DT_INT32 - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_INT32 - tensor_shape { - } - int_val: 0 - } - } - } - } - node { - name: "save/ShardedFilename" - op: "ShardedFilename" - input: "save/StringJoin" - input: "save/ShardedFilename/shard" - input: "save/num_shards" - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "save/SaveV2/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 3 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 3 - } - } - string_val: "a" - string_val: "b" - string_val: "c" - } - } - } - } - node { - name: "save/SaveV2/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 3 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 3 - } - } - string_val: "" - string_val: "" - string_val: "" - } - } - } - } - node { - name: "save/SaveV2" - op: "SaveV2" - input: "save/ShardedFilename" - input: "save/SaveV2/tensor_names" - input: "save/SaveV2/shape_and_slices" - input: "a" - input: "b" - input: "c" - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - type: DT_FLOAT - type: DT_FLOAT - } - } - } - } - node { - name: "save/control_dependency" - op: "Identity" - input: "save/ShardedFilename" - input: "^save/SaveV2" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_class" - value { - list { - s: "loc:@save/ShardedFilename" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "save/MergeV2Checkpoints/checkpoint_prefixes" - op: "Pack" - input: "save/ShardedFilename" - input: "^save/control_dependency" - attr { - key: "N" - value { - i: 1 - } - } - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "axis" - value { - i: 0 - } - } - } - node { - name: "save/MergeV2Checkpoints" - op: "MergeV2Checkpoints" - input: "save/MergeV2Checkpoints/checkpoint_prefixes" - input: "save/Const" - attr { - key: "delete_old_dirs" - value { - b: true - } - } - } - node { - name: "save/Identity" - op: "Identity" - input: "save/Const" - input: "^save/control_dependency" - input: "^save/MergeV2Checkpoints" - attr { - key: "T" - value { - type: DT_STRING - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - } - node { - name: "save/RestoreV2/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "a" - } - } - } - } - node { - name: "save/RestoreV2/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "" - } - } - } - } - node { - name: "save/RestoreV2" - op: "RestoreV2" - input: "save/Const" - input: "save/RestoreV2/tensor_names" - input: "save/RestoreV2/shape_and_slices" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - } - } - } - } - node { - name: "save/Assign" - op: "Assign" - input: "a" - input: "save/RestoreV2" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@a" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "save/RestoreV2_1/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "b" - } - } - } - } - node { - name: "save/RestoreV2_1/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "" - } - } - } - } - node { - name: "save/RestoreV2_1" - op: "RestoreV2" - input: "save/Const" - input: "save/RestoreV2_1/tensor_names" - input: "save/RestoreV2_1/shape_and_slices" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - } - } - } - } - node { - name: "save/Assign_1" - op: "Assign" - input: "b" - input: "save/RestoreV2_1" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@b" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "save/RestoreV2_2/tensor_names" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "c" - } - } - } - } - node { - name: "save/RestoreV2_2/shape_and_slices" - op: "Const" - attr { - key: "_output_shapes" - value { - list { - shape { - dim { - size: 1 - } - } - } - } - } - attr { - key: "dtype" - value { - type: DT_STRING - } - } - attr { - key: "value" - value { - tensor { - dtype: DT_STRING - tensor_shape { - dim { - size: 1 - } - } - string_val: "" - } - } - } - } - node { - name: "save/RestoreV2_2" - op: "RestoreV2" - input: "save/Const" - input: "save/RestoreV2_2/tensor_names" - input: "save/RestoreV2_2/shape_and_slices" - attr { - key: "_output_shapes" - value { - list { - shape { - unknown_rank: true - } - } - } - } - attr { - key: "dtypes" - value { - list { - type: DT_FLOAT - } - } - } - } - node { - name: "save/Assign_2" - op: "Assign" - input: "c" - input: "save/RestoreV2_2" - attr { - key: "T" - value { - type: DT_FLOAT - } - } - attr { - key: "_class" - value { - list { - s: "loc:@c" - } - } - } - attr { - key: "_output_shapes" - value { - list { - shape { - } - } - } - } - attr { - key: "use_locking" - value { - b: true - } - } - attr { - key: "validate_shape" - value { - b: true - } - } - } - node { - name: "save/restore_shard" - op: "NoOp" - input: "^save/Assign" - input: "^save/Assign_1" - input: "^save/Assign_2" - } - node { - name: "save/restore_all" - op: "NoOp" - input: "^save/restore_shard" - } - versions { - producer: 23 - } - } - saver_def { - filename_tensor_name: "save/Const:0" - save_tensor_name: "save/Identity:0" - restore_op_name: "save/restore_all" - max_to_keep: 5 - sharded: true - keep_checkpoint_every_n_hours: 10000.0 - version: V2 - } - collection_def { - key: "asset_filepaths" - value { - node_list { - value: "Const:0" - } - } - } - collection_def { - key: "legacy_init_op" - value { - node_list { - value: "group_deps" - } - } - } - collection_def { - key: "saved_model_assets" - value { - any_list { - value { - type_url: "type.googleapis.com/tensorflow.AssetFileDef" - value: "\n\t\n\007Const:0\022\007foo.txt" - } - } - } - } - collection_def { - key: "trainable_variables" - value { - bytes_list { - value: "\n\003a:0\022\010a/Assign\032\010a/read:0" - value: "\n\003b:0\022\010b/Assign\032\010b/read:0" - value: "\n\003c:0\022\010c/Assign\032\010c/read:0" - } - } - } - collection_def { - key: "variables" - value { - bytes_list { - value: "\n\003a:0\022\010a/Assign\032\010a/read:0" - value: "\n\003b:0\022\010b/Assign\032\010b/read:0" - value: "\n\003c:0\022\010c/Assign\032\010c/read:0" - } - } - } - signature_def { - key: "classify_x2_to_y3" - value { - inputs { - key: "inputs" - value { - name: "x2:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - outputs { - key: "scores" - value { - name: "y3:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/classify" - } - } - signature_def { - key: "classify_x_to_y" - value { - inputs { - key: "inputs" - value { - name: "tf_example:0" - dtype: DT_STRING - tensor_shape { - unknown_rank: true - } - } - } - outputs { - key: "scores" - value { - name: "y:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/classify" - } - } - signature_def { - key: "regress_x2_to_y3" - value { - inputs { - key: "inputs" - value { - name: "x2:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - outputs { - key: "outputs" - value { - name: "y3:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/regress" - } - } - signature_def { - key: "regress_x_to_y" - value { - inputs { - key: "inputs" - value { - name: "tf_example:0" - dtype: DT_STRING - tensor_shape { - unknown_rank: true - } - } - } - outputs { - key: "outputs" - value { - name: "y:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/regress" - } - } - signature_def { - key: "regress_x_to_y2" - value { - inputs { - key: "inputs" - value { - name: "tf_example:0" - dtype: DT_STRING - tensor_shape { - unknown_rank: true - } - } - } - outputs { - key: "outputs" - value { - name: "y2:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/regress" - } - } - signature_def { - key: "serving_default" - value { - inputs { - key: "x" - value { - name: "x:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - outputs { - key: "y" - value { - name: "y:0" - dtype: DT_FLOAT - tensor_shape { - dim { - size: -1 - } - dim { - size: 1 - } - } - } - } - method_name: "tensorflow/serving/predict" - } - } -} diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 deleted file mode 100755 index 15b75d6ef6bffc336d138d923badb3928b8c4c13..0000000000000000000000000000000000000000 Binary files a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.data-00000-of-00001 and /dev/null differ diff --git a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index b/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index deleted file mode 100755 index 7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326..0000000000000000000000000000000000000000 Binary files a/tensorflow/cc/saved_model/testdata/half_plus_two_forward_compatibility/00000123/variables/variables.index and /dev/null differ diff --git a/tensorflow/compiler/aot/compile.cc b/tensorflow/compiler/aot/compile.cc index eac8da0ab1b05e7d5cc8d27a1e1ffecc85515cdb..2b8cc6024cb85e4f6269313927ff66d1d9a1cf79 100644 --- a/tensorflow/compiler/aot/compile.cc +++ b/tensorflow/compiler/aot/compile.cc @@ -97,11 +97,15 @@ Status CompileGraph(const GraphDef& graph_def, const tf2xla::Config& config, TF_RETURN_IF_ERROR(ConvertGraphDefToXla(graph_def, config, client, &computation, &compile_result->has_context_arg)); - if (!flags.debug_dir.empty()) { + if (!flags.out_session_module.empty()) { TF_ASSIGN_OR_RETURN(std::unique_ptr module, computation.Snapshot()); - string file = io::JoinPath(flags.debug_dir, "tfcompile_xla_module.pb"); - TF_RETURN_IF_ERROR(WriteBinaryProto(Env::Default(), file, *module)); + // Serialize the SessionModule deterministically so that all the outputs of + // a tf_library genrule are deterministic. + string proto; + TF_RET_CHECK(SerializeToStringDeterministic(*module, &proto)); + TF_RETURN_IF_ERROR( + WriteStringToFile(Env::Default(), flags.out_session_module, proto)); } xla::cpu::CpuAotCompilationOptions aot_opts( flags.target_triple, flags.target_cpu, flags.target_features, diff --git a/tensorflow/compiler/aot/flags.cc b/tensorflow/compiler/aot/flags.cc index 5aff10346fa368f214436d1d0837505ffbbc771e..7c2f27e550d44c2487f91acf1029c962ac3f5d01 100644 --- a/tensorflow/compiler/aot/flags.cc +++ b/tensorflow/compiler/aot/flags.cc @@ -33,9 +33,6 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) { "fetch nodes will be dumped to stdout in a comma-separated list. " "Typically used to format arguments for other tools, e.g. " "freeze_graph."}, - {"debug_dir", &flags->debug_dir, - "Specifies a directory to dump debugging information, including " - "rewritten graphs and the XLA HLO module."}, // Flags controlling the XLA ahead-of-time compilation, that correspond to // the fields of xla::cpu::CpuAotCompilationOptions. // @@ -64,6 +61,8 @@ void AppendMainFlags(std::vector* flag_list, MainFlags* flags) { "namespaces are given, within the global namespace."}, {"out_object", &flags->out_object, "Output object file name."}, {"out_header", &flags->out_header, "Output header file name."}, + {"out_session_module", &flags->out_session_module, + "Output session module proto."}, {"gen_name_to_index", &flags->gen_name_to_index, "Generate name-to-index data for Lookup{Arg,Result}Index methods."}, {"gen_program_shape", &flags->gen_program_shape, diff --git a/tensorflow/compiler/aot/flags.h b/tensorflow/compiler/aot/flags.h index 3246dbf95c8a60130af91bc3891b15829aa5e638..3519659e3af7cd345f30080a07ce91fb858623fb 100644 --- a/tensorflow/compiler/aot/flags.h +++ b/tensorflow/compiler/aot/flags.h @@ -29,7 +29,6 @@ struct MainFlags { string graph; string config; bool dump_fetch_nodes = false; - string debug_dir; string target_triple; string target_cpu; string target_features; @@ -37,6 +36,7 @@ struct MainFlags { string cpp_class; string out_object; string out_header; + string out_session_module; // C++ codegen options bool gen_name_to_index = false; diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl index 4888760acd45f2789193884407b3742a5e9683ec..363d6925a14dfab8b79617449a73727ab55c4527 100644 --- a/tensorflow/compiler/aot/tfcompile.bzl +++ b/tensorflow/compiler/aot/tfcompile.bzl @@ -4,7 +4,7 @@ To use from your BUILD file, add the following line to load the macro: -load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") +load("@org_tensorflow//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") Then call the macro like this: @@ -16,14 +16,14 @@ tf_library( ) """ -load("//tensorflow:tensorflow.bzl", "if_android", "tf_copts") +load("@org_tensorflow//tensorflow:tensorflow.bzl", "if_android", "tf_copts") def tf_library(name, graph, config, freeze_checkpoint=None, freeze_saver=None, cpp_class=None, gen_test=True, gen_benchmark=True, visibility=None, testonly=None, tfcompile_flags=None, - tfcompile_tool="//tensorflow/compiler/aot:tfcompile", + tfcompile_tool="@org_tensorflow//tensorflow/compiler/aot:tfcompile", include_standard_runtime_deps=True, deps=None, tags=None): """Runs tfcompile to compile a TensorFlow graph into executable code. @@ -119,9 +119,9 @@ def tf_library(name, graph, config, out_nodes_file, ] + freeze_saver_srcs, outs=[freeze_file], - cmd=("$(location //tensorflow/python/tools:freeze_graph)" + + cmd=("$(location @org_tensorflow//tensorflow/python/tools:freeze_graph)" + freeze_args), - tools=["//tensorflow/python/tools:freeze_graph"], + tools=["@org_tensorflow//tensorflow/python/tools:freeze_graph"], tags=tags, ) tfcompile_graph = freeze_file @@ -165,6 +165,34 @@ def tf_library(name, graph, config, tags=tags, ) + # Rule that runs tfcompile to produce the SessionModule proto, useful for + # debugging. TODO(b/64813587): Once the SessionModule proto is + # deterministic, move this into the main rule above. + session_module_pb = name + "_session_module.pb" + native.genrule( + name=(name + "_session_module"), + srcs=[ + tfcompile_graph, + config, + ], + outs=[ + session_module_pb, + ], + cmd=("$(location " + tfcompile_tool + ")" + + " --graph=$(location " + tfcompile_graph + ")" + + " --config=$(location " + config + ")" + + " --entry_point=" + ep + + " --cpp_class=" + cpp_class + + " --target_triple=" + target_llvm_triple() + + " --out_session_module=$(@D)/" + session_module_pb + + " " + (tfcompile_flags or "")), + tools=[tfcompile_tool], + visibility=visibility, + testonly=testonly, + local=1, + tags=tags, + ) + # The cc_library rule packaging up the header and object file, and needed # kernel implementations. need_xla_data_proto = (tfcompile_flags and @@ -179,22 +207,22 @@ def tf_library(name, graph, config, # These deps are required by all tf_library targets even if # include_standard_runtime_deps is False. Without them, the # generated code will fail to compile. - "//tensorflow/compiler/tf2xla:xla_compiled_cpu_function", - "//tensorflow/core:framework_lite", + "@org_tensorflow//tensorflow/compiler/tf2xla:xla_compiled_cpu_function", + "@org_tensorflow//tensorflow/core:framework_lite", ] + (need_xla_data_proto and [ # If we're generating the program shape, we must depend on the proto. - "//tensorflow/compiler/xla:xla_data_proto", + "@org_tensorflow//tensorflow/compiler/xla:xla_data_proto", ] or []) + (include_standard_runtime_deps and [ # TODO(cwhipkey): only depend on kernel code that the model actually needed. - "//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_1d", - "//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_2d", - "//tensorflow/compiler/xla/service/cpu:cpu_runtime_avx", - "//tensorflow/compiler/xla/service/cpu:cpu_runtime_neon", - "//tensorflow/compiler/xla/service/cpu:cpu_runtime_sse4_1", - "//tensorflow/compiler/xla/service/cpu:runtime_conv2d", - "//tensorflow/compiler/xla/service/cpu:runtime_matmul", - "//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_conv2d", - "//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_matmul", + "@org_tensorflow//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_1d", + "@org_tensorflow//tensorflow/compiler/tf2xla/kernels:index_ops_kernel_argmax_float_2d", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:cpu_runtime_avx", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:cpu_runtime_neon", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:cpu_runtime_sse4_1", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_conv2d", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_matmul", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_conv2d", + "@org_tensorflow//tensorflow/compiler/xla/service/cpu:runtime_single_threaded_matmul", "//third_party/eigen3", ] or []) + (deps or []), tags=tags, @@ -220,12 +248,12 @@ def tf_library(name, graph, config, name=("gen_" + test_name), testonly=1, srcs=[ - "//tensorflow/compiler/aot:test.cc", + "@org_tensorflow//tensorflow/compiler/aot:test.cc", header_file, ], outs=[test_file], cmd=("sed " + sed_replace + - " $(location //tensorflow/compiler/aot:test.cc) " + + " $(location @org_tensorflow//tensorflow/compiler/aot:test.cc) " + "> $(OUTS)"), tags=tags, ) @@ -236,13 +264,13 @@ def tf_library(name, graph, config, srcs=[test_file], deps=[ ":" + name, - "//tensorflow/compiler/tf2xla:xla_local_runtime_context", - "//tensorflow/compiler/aot:runtime", - "//tensorflow/compiler/aot:tf_library_test_main", - "//tensorflow/compiler/xla:executable_run_options", + "@org_tensorflow//tensorflow/compiler/tf2xla:xla_local_runtime_context", + "@org_tensorflow//tensorflow/compiler/aot:runtime", + "@org_tensorflow//tensorflow/compiler/aot:tf_library_test_main", + "@org_tensorflow//tensorflow/compiler/xla:executable_run_options", "//third_party/eigen3", - "//tensorflow/core:lib", - "//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", ], tags=tags, ) @@ -250,7 +278,7 @@ def tf_library(name, graph, config, if gen_benchmark: benchmark_name = name + "_benchmark" benchmark_file = benchmark_name + ".cc" - benchmark_main = ("//tensorflow/compiler/aot:" + + benchmark_main = ("@org_tensorflow//tensorflow/compiler/aot:" + "benchmark_main.template") # Rule to rewrite benchmark.cc to produce the benchmark_file. @@ -282,13 +310,13 @@ def tf_library(name, graph, config, linkopts = if_android(["-pie", "-s"]), deps=[ ":" + name, - "//tensorflow/compiler/tf2xla:xla_local_runtime_context", - "//tensorflow/compiler/aot:benchmark", - "//tensorflow/compiler/aot:runtime", - "//tensorflow/compiler/xla:executable_run_options", + "@org_tensorflow//tensorflow/compiler/tf2xla:xla_local_runtime_context", + "@org_tensorflow//tensorflow/compiler/aot:benchmark", + "@org_tensorflow//tensorflow/compiler/aot:runtime", + "@org_tensorflow//tensorflow/compiler/xla:executable_run_options", "//third_party/eigen3", ] + if_android([ - "//tensorflow/compiler/aot:benchmark_extra_android", + "@org_tensorflow//tensorflow/compiler/aot:benchmark_extra_android", ]), tags=tags, ) @@ -298,11 +326,11 @@ def target_llvm_triple(): # TODO(toddw): Add target_triple for other targets. For details see: # http://llvm.org/docs/doxygen/html/Triple_8h_source.html return select({ - "//tensorflow:android_armeabi": "armv5-none-android", - "//tensorflow:android_arm": "armv7-none-android", - "//tensorflow:android_arm64": "aarch64-none-android", - "//tensorflow:android_x86": "i686-none-android", - "//tensorflow:linux_ppc64le": "ppc64le-ibm-linux-gnu", - "//tensorflow:darwin": "x86_64-none-darwin", + "@org_tensorflow//tensorflow:android_armeabi": "armv5-none-android", + "@org_tensorflow//tensorflow:android_arm": "armv7-none-android", + "@org_tensorflow//tensorflow:android_arm64": "aarch64-none-android", + "@org_tensorflow//tensorflow:android_x86": "i686-none-android", + "@org_tensorflow//tensorflow:linux_ppc64le": "ppc64le-ibm-linux-gnu", + "@org_tensorflow//tensorflow:darwin": "x86_64-none-darwin", "//conditions:default": "x86_64-pc-linux", }) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index 579ce415c5c3c4951be1596a37d47b7930bcf4fb..b3d258aea177fbefa4bae51d8156da2ff86c9032 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -144,8 +144,8 @@ TEST(XlaCompilationTest, UnsupportedTypes) { Node* a = ops::SourceOp( "Const", builder.opts() .WithName("A") - .WithAttr("dtype", DT_COMPLEX64) - .WithAttr("value", Tensor(DT_COMPLEX64, TensorShape()))); + .WithAttr("dtype", DT_COMPLEX128) + .WithAttr("value", Tensor(DT_COMPLEX128, TensorShape()))); Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B")); ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C")); TF_EXPECT_OK(builder.ToGraph(graph.get())); diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc index 2e33fdca657f470270cb25fa2ac661a441b70552..e238252751e677eb947f6df03e3b2f2e948ffe19 100644 --- a/tensorflow/compiler/jit/xla_cpu_device.cc +++ b/tensorflow/compiler/jit/xla_cpu_device.cc @@ -50,8 +50,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_CPU, XlaCpuDeviceFactory); // Kernel registrations -constexpr std::array kAllXlaCpuTypes = { - {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kAllXlaCpuTypes = { + {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}}; REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_CPU, XlaLocalLaunchOp, kAllXlaCpuTypes); REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_CPU, kAllXlaCpuTypes); diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index a2c91511ec17e2770601f5d5389ccbff96f81716..d4d8fe1c1d575b4e35d624621cc709e3a16569d5 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/common_runtime/device_factory.h" #include "tensorflow/core/common_runtime/dma_helper.h" #include "tensorflow/core/common_runtime/function.h" +#include "tensorflow/core/common_runtime/renamed_device.h" #include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/framework/device_base.h" #include "tensorflow/core/framework/function.h" @@ -161,7 +162,8 @@ const DeviceType& XlaDevice::Metadata::jit_device_type() const { /* static */ Status XlaDevice::GetMetadata(OpKernelContext* ctx, const Metadata** metadata) { - XlaDevice* xla_device = dynamic_cast(ctx->device()); + XlaDevice* xla_device = + dynamic_cast(ctx->device()->UnderlyingDevice()); if (xla_device == nullptr) { return errors::Internal( "Cannot get XLA metadata from non-XLA device \"", ctx->device()->name(), @@ -239,7 +241,8 @@ void XlaDevice::Compute(OpKernel* op_kernel, OpKernelContext* context) { // When TraceMe profiling is off (which is the default), the // following TraceMe constructor is simply a conditional test of // false value. Measurements show that its overhead is negligible. - port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); op_kernel->Compute(context); } @@ -247,7 +250,8 @@ void XlaDevice::ComputeAsync(AsyncOpKernel* op_kernel, OpKernelContext* context, AsyncOpKernel::DoneCallback done) { VLOG(1) << "XlaDevice::ComputeAsync " << op_kernel->name() << ":" << op_kernel->type_string(); - port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); op_kernel->ComputeAsync(context, done); } diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc index 5233665ec283a770117aa5bec1c0d01f17a04526..2326070358d67c0cf30ef17fab5c93862cd8932c 100644 --- a/tensorflow/compiler/jit/xla_gpu_device.cc +++ b/tensorflow/compiler/jit/xla_gpu_device.cc @@ -55,8 +55,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_GPU, XlaGpuDeviceFactory); // Kernel registrations -constexpr std::array kAllXlaGpuTypes = { - {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kAllXlaGpuTypes = { + {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}}; REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_GPU, XlaLocalLaunchOp, kAllXlaGpuTypes); REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_GPU, kAllXlaGpuTypes); diff --git a/tensorflow/compiler/plugin/BUILD b/tensorflow/compiler/plugin/BUILD index f0886721546bba3ace76e50608dc4fe61416da5c..c1edf2448c54ffddd7b70dcdfb1609080ca81b65 100644 --- a/tensorflow/compiler/plugin/BUILD +++ b/tensorflow/compiler/plugin/BUILD @@ -40,3 +40,17 @@ cc_library( #"//tensorflow/compiler/plugin/example:example_lib", ], ) + +#----------------------------------------------------------------------------- + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 0eed475140c72034ad664b3ae03f09944d92473f..0ff99c5156ded2ae05c6976e3da8f31fce32f8f2 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -23,6 +23,10 @@ load("//tensorflow:tensorflow.bzl", "cuda_py_test") load("//tensorflow/compiler/aot:tfcompile.bzl", "tf_library") load("//tensorflow/compiler/tests:build_defs.bzl", "tf_xla_py_test") load("//tensorflow/compiler/tests:build_defs.bzl", "generate_backend_suites") +load( + "//tensorflow/core:platform/default/build_config_root.bzl", + "tf_cuda_tests_tags", +) generate_backend_suites() @@ -460,7 +464,7 @@ tf_xla_py_test( tf_xla_py_test( name = "unary_ops_test", - size = "small", + size = "medium", srcs = ["unary_ops_test.py"], deps = [ ":xla_test", @@ -581,11 +585,12 @@ cc_library( tf_cuda_cc_test( name = "randomized_tests", + size = "large", # This test is randomized, so only run it if explicitly requested. tags = [ "manual", "notap", - ], + ] + tf_cuda_tests_tags(), deps = [":randomized_tests_library"], ) diff --git a/tensorflow/compiler/tests/argminmax_test.py b/tensorflow/compiler/tests/argminmax_test.py index c2ce121348da034efe002dd8db0f5b0703324a41..ec547e16cd9c91a1e25bc963b9a3cafddf7326cd 100644 --- a/tensorflow/compiler/tests/argminmax_test.py +++ b/tensorflow/compiler/tests/argminmax_test.py @@ -46,7 +46,9 @@ class ArgMinMaxTest(xla_test.XLATestCase): self.assertAllEqual(result, expected) def testArgMinMax(self): - for dtype in self.numeric_types: + # Complex numbers do not support argmin/argmax. + minmax_types = set(self.numeric_types) - set(self.complex_types) + for dtype in minmax_types: self._assertOpOutputMatchesExpected( lambda x: math_ops.argmax(x, axis=0, output_type=dtypes.int32), np.array([1, 10, 27, 3, 3, 4], dtype=dtype), diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py index 44b32b1668443f65ee0a47766683e2730d64b929..d412c572ae16b84c2434819aa0a2d881defef5f9 100644 --- a/tensorflow/compiler/tests/binary_ops_test.py +++ b/tensorflow/compiler/tests/binary_ops_test.py @@ -94,6 +94,15 @@ class BinaryOpsTest(XLATestCase): dtype(4), expected=np.array([[16], [81]], dtype=dtype)) + atan2_supported = self.device == "XLA_GPU" + if atan2_supported: + self._testBinary( + math_ops.atan2, + np.array([0, np.sqrt(2), 1, np.sqrt(2), 0], dtype), + np.array([1, np.sqrt(2), 0, -np.sqrt(2), -1], dtype), + expected=np.array( + [0, np.pi / 4, np.pi / 2, np.pi * 3 / 4, np.pi], dtype=dtype)) + self._testBinary( gen_math_ops._reciprocal_grad, np.array([4, -3, -2, 1], dtype=dtype), @@ -209,6 +218,22 @@ class BinaryOpsTest(XLATestCase): np.array([0b0, 0b101, 0b1001], dtype=dtype), expected=np.array([0b1, 0b101, 0b1001], dtype=dtype)) + lhs = np.array([0, 5, 3, 14], dtype=dtype) + rhs = np.array([5, 0, 7, 11], dtype=dtype) + self._testBinary( + bitwise_ops.left_shift, lhs, rhs, + expected=np.left_shift(lhs, rhs)) + self._testBinary( + bitwise_ops.right_shift, lhs, rhs, + expected=np.right_shift(lhs, rhs)) + + if dtype in [np.int8, np.int16, np.int32, np.int64]: + lhs = np.array([-1, -5, -3, -14], dtype=dtype) + rhs = np.array([5, 0, 1, 11], dtype=dtype) + self._testBinary( + bitwise_ops.right_shift, lhs, rhs, + expected=np.right_shift(lhs, rhs)) + def testNumericOps(self): for dtype in self.numeric_types: self._testBinary( @@ -243,37 +268,38 @@ class BinaryOpsTest(XLATestCase): dtype(7), expected=np.array([[-6], [-5]], dtype=dtype)) - self._testBinary( - math_ops.maximum, - np.array([1, 2], dtype=dtype), - np.array([10, 20], dtype=dtype), - expected=np.array([10, 20], dtype=dtype)) - self._testBinary( - math_ops.maximum, - dtype(5), - np.array([1, 20], dtype=dtype), - expected=np.array([5, 20], dtype=dtype)) - self._testBinary( - math_ops.maximum, - np.array([[10], [2]], dtype=dtype), - dtype(7), - expected=np.array([[10], [7]], dtype=dtype)) + if dtype not in self.complex_types: # min/max not supported for complex + self._testBinary( + math_ops.maximum, + np.array([1, 2], dtype=dtype), + np.array([10, 20], dtype=dtype), + expected=np.array([10, 20], dtype=dtype)) + self._testBinary( + math_ops.maximum, + dtype(5), + np.array([1, 20], dtype=dtype), + expected=np.array([5, 20], dtype=dtype)) + self._testBinary( + math_ops.maximum, + np.array([[10], [2]], dtype=dtype), + dtype(7), + expected=np.array([[10], [7]], dtype=dtype)) - self._testBinary( - math_ops.minimum, - np.array([1, 20], dtype=dtype), - np.array([10, 2], dtype=dtype), - expected=np.array([1, 2], dtype=dtype)) - self._testBinary( - math_ops.minimum, - dtype(5), - np.array([1, 20], dtype=dtype), - expected=np.array([1, 5], dtype=dtype)) - self._testBinary( - math_ops.minimum, - np.array([[10], [2]], dtype=dtype), - dtype(7), - expected=np.array([[7], [2]], dtype=dtype)) + self._testBinary( + math_ops.minimum, + np.array([1, 20], dtype=dtype), + np.array([10, 2], dtype=dtype), + expected=np.array([1, 2], dtype=dtype)) + self._testBinary( + math_ops.minimum, + dtype(5), + np.array([1, 20], dtype=dtype), + expected=np.array([1, 5], dtype=dtype)) + self._testBinary( + math_ops.minimum, + np.array([[10], [2]], dtype=dtype), + dtype(7), + expected=np.array([[7], [2]], dtype=dtype)) self._testBinary( math_ops.multiply, @@ -291,21 +317,23 @@ class BinaryOpsTest(XLATestCase): dtype(7), expected=np.array([[70], [14]], dtype=dtype)) - self._testBinary( - math_ops.squared_difference, - np.array([1, 2], dtype=dtype), - np.array([10, 20], dtype=dtype), - expected=np.array([81, 324], dtype=dtype)) - self._testBinary( - math_ops.squared_difference, - dtype(5), - np.array([1, 2], dtype=dtype), - expected=np.array([16, 9], dtype=dtype)) - self._testBinary( - math_ops.squared_difference, - np.array([[1], [2]], dtype=dtype), - dtype(7), - expected=np.array([[36], [25]], dtype=dtype)) + # Complex support for squared_difference is incidental, see b/68205550 + if dtype not in self.complex_types: + self._testBinary( + math_ops.squared_difference, + np.array([1, 2], dtype=dtype), + np.array([10, 20], dtype=dtype), + expected=np.array([81, 324], dtype=dtype)) + self._testBinary( + math_ops.squared_difference, + dtype(5), + np.array([1, 2], dtype=dtype), + expected=np.array([16, 9], dtype=dtype)) + self._testBinary( + math_ops.squared_difference, + np.array([[1], [2]], dtype=dtype), + dtype(7), + expected=np.array([[36], [25]], dtype=dtype)) self._testBinary( nn_ops.bias_add, @@ -318,6 +346,139 @@ class BinaryOpsTest(XLATestCase): np.array([2, -1], dtype=dtype), expected=np.array([[[[3, 1], [5, 3]]]], dtype=dtype)) + def testComplexOps(self): + for dtype in self.complex_types: + ctypes = {np.complex64: np.float32} + self._testBinary( + math_ops.complex, + np.array([[[[-1, 2], [2, 0]]]], dtype=ctypes[dtype]), + np.array([[[[2, -3], [0, 4]]]], dtype=ctypes[dtype]), + expected=np.array([[[[-1 + 2j, 2 - 3j], [2, 4j]]]], dtype=dtype)) + + self._testBinary( + lambda x, y: math_ops.approximate_equal(x, y, tolerance=0.0001), + np.array( + [[[[-1 + 2j, 2.00009999 - 3j], [2 - 3j, 3 + 4.01j]]]], + dtype=dtype), + np.array( + [[[[-1.001 + 2j, 2 - 3j], [2 - 3.00009j, 3 + 4j]]]], dtype=dtype), + expected=np.array([[[[False, True], [True, False]]]], dtype=dtype)) + + self._testBinary( + gen_math_ops._real_div, + np.array([3, 3j, -1.5j, -8, 2 + 3j, 2 + 4j, 44 + 3j], dtype=dtype), + np.array([2, -2, 7j, -4j, 4 - 6j, 1 + 2j, 0], dtype=dtype), + expected=np.array( + [ + 1.5, -1.5j, -0.2142857, -2j, (2 + 3j) / (4 - 6j), 2, + float("inf") + ], + dtype=dtype)) + + # TODO(b/65408531): support+test pow for cplx + + lhs = np.array([4 + 2j, -3 - 1j, 2j, 1], dtype=dtype) + rhs = np.array([5, -6j, 7 - 3j, -8j], dtype=dtype) + self._testBinary( + gen_math_ops._reciprocal_grad, lhs, rhs, expected=-rhs * lhs * lhs) + + self._testBinary( + gen_math_ops._sigmoid_grad, lhs, rhs, expected=rhs * lhs * (1 - lhs)) + + # TODO(b/65408531): support+test _rsqrt_grad for cplx (needs pow) + + self._testBinary( + gen_math_ops._sqrt_grad, lhs, rhs, expected=rhs / (2 * lhs)) + + self._testBinary( + gen_math_ops._tanh_grad, lhs, rhs, expected=rhs * (1 - lhs * lhs)) + + def testComplexMath(self): + for dtype in self.complex_types: + self._testBinary( + math_ops.add, + np.array([1 + 3j, 2 + 7j], dtype=dtype), + np.array([10 - 4j, 20 + 17j], dtype=dtype), + expected=np.array([11 - 1j, 22 + 24j], dtype=dtype)) + self._testBinary( + math_ops.add, + dtype(5 - 7j), + np.array([1 + 2j, 2 + 4j], dtype=dtype), + expected=np.array([6 - 5j, 7 - 3j], dtype=dtype)) + self._testBinary( + math_ops.add, + np.array([[1 - 2j], [2 + 1j]], dtype=dtype), + dtype(7 + 5j), + expected=np.array([[8 + 3j], [9 + 6j]], dtype=dtype)) + + self._testBinary( + math_ops.subtract, + np.array([1 + 3j, 2 + 7j], dtype=dtype), + np.array([10 - 4j, 20 + 17j], dtype=dtype), + expected=np.array([-9 + 7j, -18 - 10j], dtype=dtype)) + self._testBinary( + math_ops.subtract, + dtype(5 - 7j), + np.array([1 + 2j, 2 + 4j], dtype=dtype), + expected=np.array([4 - 9j, 3 - 11j], dtype=dtype)) + self._testBinary( + math_ops.subtract, + np.array([[1 - 2j], [2 + 1j]], dtype=dtype), + dtype(7 + 5j), + expected=np.array([[-6 - 7j], [-5 - 4j]], dtype=dtype)) + + self._testBinary( + math_ops.multiply, + np.array([1 + 3j, 2 + 7j], dtype=dtype), + np.array([10 - 4j, 20 + 17j], dtype=dtype), + expected=np.array( + [(1 + 3j) * (10 - 4j), (2 + 7j) * (20 + 17j)], dtype=dtype)) + self._testBinary( + math_ops.multiply, + dtype(5 - 7j), + np.array([1 + 2j, 2 + 4j], dtype=dtype), + expected=np.array( + [(5 - 7j) * (1 + 2j), (5 - 7j) * (2 + 4j)], dtype=dtype)) + self._testBinary( + math_ops.multiply, + np.array([[1 - 2j], [2 + 1j]], dtype=dtype), + dtype(7 + 5j), + expected=np.array( + [[(7 + 5j) * (1 - 2j)], [(7 + 5j) * (2 + 1j)]], dtype=dtype)) + + self._testBinary( + math_ops.div, + np.array([8 - 1j, 2 + 16j], dtype=dtype), + np.array([2 + 4j, 4 - 8j], dtype=dtype), + expected=np.array( + [(8 - 1j) / (2 + 4j), (2 + 16j) / (4 - 8j)], dtype=dtype)) + self._testBinary( + math_ops.div, + dtype(1 + 2j), + np.array([2 + 4j, 4 - 8j], dtype=dtype), + expected=np.array( + [(1 + 2j) / (2 + 4j), (1 + 2j) / (4 - 8j)], dtype=dtype)) + self._testBinary( + math_ops.div, + np.array([2 + 4j, 4 - 8j], dtype=dtype), + dtype(1 + 2j), + expected=np.array( + [(2 + 4j) / (1 + 2j), (4 - 8j) / (1 + 2j)], dtype=dtype)) + + # TODO(b/68205550): math_ops.squared_difference shouldn't be supported. + + self._testBinary( + nn_ops.bias_add, + np.array([[1 + 2j, 2 + 7j], [3 - 5j, 4 + 2j]], dtype=dtype), + np.array([2 + 6j, -1 - 3j], dtype=dtype), + expected=np.array([[3 + 8j, 1 + 4j], [5 + 1j, 3 - 1j]], dtype=dtype)) + self._testBinary( + nn_ops.bias_add, + np.array([[[[1 + 4j, 2 - 1j], [3 + 7j, 4]]]], dtype=dtype), + np.array([2 + 1j, -1 + 2j], dtype=dtype), + expected=np.array( + [[[[3 + 5j, 1 + 1j], [5 + 8j, 3 + 2j]]]], dtype=dtype)) + def _testDivision(self, dtype): """Test cases for division operators.""" self._testBinary( @@ -336,18 +497,19 @@ class BinaryOpsTest(XLATestCase): dtype(2), expected=np.array([[5], [2]], dtype=dtype)) - self._testBinary( - gen_math_ops._floor_div, - np.array([3, 3, -1, -9, -8], dtype=dtype), - np.array([2, -2, 7, 2, -4], dtype=dtype), - expected=np.array([1, -2, -1, -5, 2], dtype=dtype)) + if dtype not in self.complex_types: # floordiv unsupported for complex. + self._testBinary( + gen_math_ops._floor_div, + np.array([3, 3, -1, -9, -8], dtype=dtype), + np.array([2, -2, 7, 2, -4], dtype=dtype), + expected=np.array([1, -2, -1, -5, 2], dtype=dtype)) def testIntDivision(self): for dtype in self.int_types: self._testDivision(dtype) def testFloatDivision(self): - for dtype in self.float_types: + for dtype in self.float_types + self.complex_types: self._testDivision(dtype) def _testRemainder(self, dtype): @@ -691,6 +853,20 @@ class BinaryOpsTest(XLATestCase): [0, 0, 0, 0, 0, 0]], dtype=dtype)) + self._testBinary( + lambda x, y: array_ops.pad(x, y, constant_values=7), + np.array( + [[1, 2, 3], [4, 5, 6]], dtype=dtype), + np.array( + [[0, 3], [2, 1]], dtype=np.int32), + expected=np.array( + [[7, 7, 1, 2, 3, 7], + [7, 7, 4, 5, 6, 7], + [7, 7, 7, 7, 7, 7], + [7, 7, 7, 7, 7, 7], + [7, 7, 7, 7, 7, 7]], + dtype=dtype)) + def testMirrorPad(self): mirror_pad = lambda t, paddings: array_ops.pad(t, paddings, "REFLECT") for dtype in self.numeric_types: diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl index a56c53de0fb5f76c94064e2bdc2f1a543a207b09..0528a5415d579a844e68403ace1bb8982a10a841 100644 --- a/tensorflow/compiler/tests/build_defs.bzl +++ b/tensorflow/compiler/tests/build_defs.bzl @@ -49,11 +49,15 @@ def tf_xla_py_test(name, srcs=[], deps=[], tags=[], data=[], main=None, backend_deps = [] backend_data = [] if backend == "cpu": - backend_args += ["--test_device=XLA_CPU", - "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL"] + backend_args += [ + "--test_device=XLA_CPU", + "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64" + ] elif backend == "gpu": - backend_args += ["--test_device=XLA_GPU", - "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL"] + backend_args += [ + "--test_device=XLA_GPU", + "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64" + ] backend_tags += ["requires-gpu-sm35"] elif backend in plugins: backend_args += ["--test_device=" + plugins[backend]["device"], diff --git a/tensorflow/compiler/tests/gather_test.py b/tensorflow/compiler/tests/gather_test.py index 4b81c1d7abcb89ef8f776137d1c7d57481c82515..664c77f2000281e3be989665664c1be58d4dd1e5 100644 --- a/tensorflow/compiler/tests/gather_test.py +++ b/tensorflow/compiler/tests/gather_test.py @@ -30,8 +30,6 @@ from tensorflow.python.platform import test FLAGS = flags.FLAGS -_TEST_TYPES = [dtypes.float32] - class GatherTest(xla_test.XLATestCase): @@ -46,7 +44,7 @@ class GatherTest(xla_test.XLATestCase): def testScalar1D(self): with self.test_session() as session, self.test_scope(): data = np.array([0, 1, 2, 3, 7, 5]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for indices in 4, [1, 2, 2, 4, 5]: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -60,7 +58,7 @@ class GatherTest(xla_test.XLATestCase): with self.test_session() as session, self.test_scope(): data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], [12, 13, 14]]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, -1: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -74,7 +72,7 @@ class GatherTest(xla_test.XLATestCase): with self.test_session() as session, self.test_scope(): data = np.array([[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11], [12, 13, 14]]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, -1: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -94,7 +92,7 @@ class GatherTest(xla_test.XLATestCase): [12, 13, 14]]) # The indices must be in bounds for any axis. indices_np = np.array([0, 1, 0, 2]) - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, -1: params_np = self._buildParams(data, dtype) params = array_ops.placeholder(dtype=dtype) @@ -112,7 +110,7 @@ class GatherTest(xla_test.XLATestCase): """Check that scalar and empty indices shapes work as well.""" shape = (2, 1, 3, 2) for indices_shape in (), (0,), (2, 0), (2, 3): - for dtype in _TEST_TYPES: + for dtype in self.all_tf_types: for axis in 0, 1, 2, 3, -1, -2: params = self._buildParams(np.random.randn(*shape), dtype) indices = np.random.randint(shape[axis], size=indices_shape) diff --git a/tensorflow/compiler/tests/nary_ops_test.py b/tensorflow/compiler/tests/nary_ops_test.py index ae60d78f1a8dd898c5428a82be2196b52d4638d8..e4843b169b943b63346b783ddc50039030988ca5 100644 --- a/tensorflow/compiler/tests/nary_ops_test.py +++ b/tensorflow/compiler/tests/nary_ops_test.py @@ -68,6 +68,26 @@ class NAryOpsTest(XLATestCase): np.array([42], dtype=np.float32)], expected=np.array([48], dtype=np.float32)) + def testComplex(self): + for dtype in self.complex_types: + self._testNAry( + math_ops.add_n, [np.array([[1 + 2j, 2 - 3j, 3 + 4j]], dtype=dtype)], + expected=np.array([[1 + 2j, 2 - 3j, 3 + 4j]], dtype=dtype)) + + self._testNAry( + math_ops.add_n, [ + np.array([1 + 2j, 2 - 3j], dtype=dtype), + np.array([10j, 20], dtype=dtype) + ], + expected=np.array([1 + 12j, 22 - 3j], dtype=dtype)) + self._testNAry( + math_ops.add_n, [ + np.array([-4, 5j], dtype=dtype), + np.array([2 + 10j, -2], dtype=dtype), + np.array([42j, 3 + 3j], dtype=dtype) + ], + expected=np.array([-2 + 52j, 1 + 8j], dtype=dtype)) + @unittest.skip("IdentityN is temporarily CompilationOnly as workaround") def testIdentityN(self): self._testNAryLists(array_ops.identity_n, diff --git a/tensorflow/compiler/tests/random_ops_test.py b/tensorflow/compiler/tests/random_ops_test.py index a17a3f3d6536eea780106d84bcf4ce92c0fd017e..d6c93088d4efff7d8306e262a79ae49d3d8ac722 100644 --- a/tensorflow/compiler/tests/random_ops_test.py +++ b/tensorflow/compiler/tests/random_ops_test.py @@ -29,6 +29,9 @@ from tensorflow.python.platform import googletest class RandomOpsTest(XLATestCase): """Test cases for random-number generating operators.""" + def _random_types(self): + return set(self.numeric_types) - set(self.complex_types) + def _testRngIsNotConstant(self, rng, dtype): # Tests that 'rng' does not always return the same value. with self.test_session() as sess: @@ -51,7 +54,8 @@ class RandomOpsTest(XLATestCase): def rng(dtype): return random_ops.random_uniform(shape=[2], dtype=dtype, maxval=1000000) - for dtype in self.numeric_types: + + for dtype in self._random_types(): self._testRngIsNotConstant(rng, dtype) def testRandomNormalIsNotConstant(self): @@ -63,7 +67,7 @@ class RandomOpsTest(XLATestCase): self._testRngIsNotConstant(rng, dtype) def testRandomUniformIsInRange(self): - for dtype in self.numeric_types: + for dtype in self._random_types(): with self.test_session() as sess: with self.test_scope(): x = random_ops.random_uniform(shape=[1000], dtype=dtype, minval=-2, diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc index 5129171cd42b09f31bb1a4da02ffc6be6093f6f1..6a8c3bcd55a6e454a19b6249cf4eb48739c8657f 100644 --- a/tensorflow/compiler/tests/randomized_tests.cc +++ b/tensorflow/compiler/tests/randomized_tests.cc @@ -75,7 +75,7 @@ namespace { // Command line flags: see main() below. int64 tf_xla_random_seed = 0; int32 tf_xla_test_repetitions = 20; -int64 tf_xla_max_tensor_size = 100000LL; +int64 tf_xla_max_tensor_size = 10000LL; string* tf_xla_test_device_ptr; // initial value set in main() bool tf_xla_test_use_jit = true; @@ -83,8 +83,8 @@ string LocalDeviceToFullDeviceName(const string& device) { return strings::StrCat("/job:localhost/replica:0/task:0/device:", device); } -constexpr std::array kAllXlaTypes = { - {DT_INT32, DT_FLOAT, DT_BOOL}}; +constexpr std::array kAllXlaTypes = { + {DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}}; // An OpTestBuilder is a graph builder class that takes as input an operator to // test, its inputs and attributes, and builds a graph that executes the @@ -367,11 +367,11 @@ OpTest::OpTest() { void OpTest::Repeatedly(const std::function& fn) { int const max_repetitions = tf_xla_test_repetitions; int valid_test_runs = 0; - // We run up to 20 * max_repetitions times; the idea is that if we roll the + // We run up to 100 * max_repetitions times; the idea is that if we roll the // dice enough times we will find some valid parameters. We want to put an // upper limit on the number iterations just in case the probability of // finding feasible parameters is very low. - for (int i = 0; !HasFailure() && i < max_repetitions * 20 && + for (int i = 0; !HasFailure() && i < max_repetitions * 100 && valid_test_runs < max_repetitions; ++i) { TestResult result = fn(); @@ -449,6 +449,13 @@ Tensor OpTest::RandomTensor(DataType dtype, gtl::ArraySlice shape) { }); break; } + case DT_COMPLEX64: { + std::uniform_real_distribution distribution(-1.0f, 1.0f); + test::FillFn(&tensor, [this, &distribution](int i) { + return complex64(distribution(generator()), distribution(generator())); + }); + break; + } case DT_INT32: { std::uniform_int_distribution distribution(-(1 << 20), 1 << 20); test::FillFn(&tensor, [this, &distribution](int i) -> int32 { @@ -624,11 +631,47 @@ std::vector OpTest::AsInt32s(const std::vector& int64s) { // Functions for comparing tensors. +template +double Abs(T x) { + return std::fabs(x); +} + +template <> +double Abs(complex64 x) { + return std::abs(x); +} + template bool IsClose(const T& x, const T& y, double atol, double rtol) { if (std::isnan(x) && std::isnan(y)) return true; if (x == y) return true; // Allow inf == inf. - return fabs(x - y) < atol + rtol * fabs(x); + return Abs(x - y) < atol + rtol * Abs(x); +} + +template <> +bool IsClose(const complex64& x, const complex64& y, double atol, + double rtol) { + if (std::isnan(x.real()) && std::isnan(y.real())) { + if (std::isnan(x.imag()) && std::isnan(y.imag())) { + return true; + } + if (x.imag() == y.imag()) return true; // Allow inf == inf. + return Abs(x.imag() - y.imag()) < atol + rtol * Abs(x.imag()); + } else if (std::isnan(x.imag()) && std::isnan(y.imag())) { + if (x.real() == y.real()) return true; // Allow inf == inf. + return Abs(x.real() - y.real()) < atol + rtol * Abs(x.real()); + } + if (x == y) return true; // Allow inf == inf. + return Abs(x - y) < atol + rtol * Abs(x); +} + +template +string Str(T x) { + return strings::StrCat(x); +} +template <> +string Str(complex64 x) { + return strings::StrCat("(", x.real(), ", ", x.imag(), ")"); } template @@ -639,9 +682,10 @@ Status TensorsAreCloseImpl(const Tensor& x, const Tensor& y, double atol, for (int i = 0; i < Tx.size(); ++i) { if (!IsClose(Tx(i), Ty(i), atol, rtol)) { return errors::InvalidArgument(strings::StrCat( - i, "-th tensor element isn't close: ", Tx(i), " vs. ", Ty(i), - ". x = ", x.DebugString(), "y = ", y.DebugString(), "atol = ", atol, - " rtol = ", rtol, " tol = ", atol + rtol * std::fabs(Tx(i)))); + i, "-th tensor element isn't close: ", Str(Tx(i)), " vs. ", + Str(Ty(i)), ". x = ", x.DebugString(), "y = ", y.DebugString(), + "atol = ", atol, " rtol = ", rtol, + " tol = ", atol + rtol * Abs(Tx(i)))); } } return Status::OK(); @@ -683,6 +727,8 @@ Status TensorsAreClose(const Tensor& a, const Tensor& b, double atol, return TensorsAreCloseImpl(a, b, atol, rtol); case DT_DOUBLE: return TensorsAreCloseImpl(a, b, atol, rtol); + case DT_COMPLEX64: + return TensorsAreCloseImpl(a, b, atol, rtol); case DT_INT32: return TensorsAreEqualImpl(a, b); case DT_INT64: @@ -822,7 +868,7 @@ Tensor AsIntTensor(DataType dtype, const std::vector& values) { TEST_F(OpTest, Abs) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Abs").RandomInput(type).Attr("T", type)); }); @@ -837,7 +883,7 @@ TEST_F(OpTest, Acosh) { TEST_F(OpTest, Add) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Add") .RandomInput(type, dims.first) @@ -848,7 +894,7 @@ TEST_F(OpTest, Add) { TEST_F(OpTest, AddN) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); int n = std::uniform_int_distribution(1, 5)(generator()); auto shape = RandomDims(); @@ -875,6 +921,14 @@ TEST_F(OpTest, All) { }); } +TEST_F(OpTest, Angle) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Angle") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, Any) { Repeatedly([this]() { std::vector data_dims = RandomDims(); @@ -889,10 +943,11 @@ TEST_F(OpTest, Any) { TEST_F(OpTest, ApproximateEqual) { Repeatedly([this]() { - auto dims = RandomDims(); + auto dims = BroadcastableDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ApproximateEqual") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) + .RandomInput(type, dims.first) + .RandomInput(type, dims.second) .Attr("T", DT_FLOAT)); }); } @@ -943,6 +998,16 @@ TEST_F(OpTest, Atanh) { }); } +TEST_F(OpTest, Atan2) { + Repeatedly([this]() { + auto dims = BroadcastableDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Atan2") + .RandomInput(DT_FLOAT, dims.first) + .RandomInput(DT_FLOAT, dims.second) + .Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, AvgPool) { Repeatedly([this]() { std::uniform_int_distribution random_int(1, 5); @@ -1038,6 +1103,7 @@ TEST_F(OpTest, AvgPool3DGrad) { TEST_F(OpTest, BatchMatMul) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); std::vector output_dims = RandomDims(2, 5, 0, 7); int64 ndims = output_dims.size(); int64 inner_dim = RandomDim(); @@ -1056,9 +1122,9 @@ TEST_F(OpTest, BatchMatMul) { } return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchMatMul") - .RandomInput(DT_FLOAT, x_dims) - .RandomInput(DT_FLOAT, y_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, x_dims) + .RandomInput(type, y_dims) + .Attr("T", type) .Attr("adj_x", adj_x) .Attr("adj_y", adj_y)); }); @@ -1090,10 +1156,11 @@ TEST_F(OpTest, BatchToSpace) { CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals), TensorShape({num_block_dims, 2}))); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BatchToSpace") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(crops) - .Attr("T", DT_FLOAT) + .Attr("T", type) .Attr("block_size", block_size)); }); } @@ -1127,13 +1194,14 @@ TEST_F(OpTest, BatchToSpaceND) { CHECK(crops.CopyFrom(AsIntTensor(DT_INT32, crop_vals), TensorShape({num_block_dims, 2}))); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("BatchToSpaceND") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(test::AsTensor( std::vector(block_dims.begin(), block_dims.end()))) .Input(crops) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } @@ -1142,18 +1210,20 @@ TEST_F(OpTest, BiasAdd) { auto x_dims = RandomDims(2, kDefaultMaxRank); auto y_dims = {x_dims[x_dims.size() - 1]}; // TODO(phawkins): test both data formats. + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAdd") - .RandomInput(DT_FLOAT, x_dims) - .RandomInput(DT_FLOAT, y_dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, x_dims) + .RandomInput(type, y_dims) + .Attr("T", type)); }); } TEST_F(OpTest, BiasAddGrad) { Repeatedly([this]() { // TODO(phawkins): test both data formats. + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("BiasAddGrad").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("BiasAddGrad").RandomInput(type).Attr("T", type)); }); } @@ -1161,10 +1231,11 @@ TEST_F(OpTest, BiasAddV1) { Repeatedly([this]() { auto x_dims = RandomDims(2, kDefaultMaxRank); auto y_dims = {x_dims[x_dims.size() - 1]}; + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("BiasAddV1") - .RandomInput(DT_FLOAT, x_dims) - .RandomInput(DT_FLOAT, y_dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, x_dims) + .RandomInput(type, y_dims) + .Attr("T", type)); }); } @@ -1193,7 +1264,7 @@ TEST_F(OpTest, BitwiseOr) { TEST_F(OpTest, BroadcastArgs) { Repeatedly([this]() { // TODO(phawkins): only int32 seems to be implemented in Tensorflow. - // DataType type = Choose({DT_INT32, DT_INT64}); + // auto type = Choose({DT_INT32, DT_INT64}); DataType type = DT_INT32; auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose( @@ -1207,7 +1278,7 @@ TEST_F(OpTest, BroadcastArgs) { TEST_F(OpTest, BroadcastGradientArgs) { Repeatedly([this]() { // TODO(phawkins): only int32 seems to be implemented in Tensorflow. - // DataType type = Choose({DT_INT32, DT_INT64}); + // auto type = Choose({DT_INT32, DT_INT64}); DataType type = DT_INT32; auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose( @@ -1221,8 +1292,8 @@ TEST_F(OpTest, BroadcastGradientArgs) { TEST_F(OpTest, Cast) { Repeatedly([this]() { DataType src_type, dst_type; - src_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL}); - dst_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL}); + src_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}); + dst_type = Choose({DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Cast") .RandomInput(src_type) .Attr("SrcT", src_type) @@ -1237,9 +1308,19 @@ TEST_F(OpTest, Ceil) { }); } +TEST_F(OpTest, Complex) { + Repeatedly([this]() { + auto dims = BroadcastableDims(); + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Complex") + .RandomInput(DT_FLOAT, dims.first) + .RandomInput(DT_FLOAT, dims.second) + .Attr("T", DT_FLOAT)); + }); +} + TEST_F(OpTest, Concat) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(2, 5)(generator()); std::vector dims = RandomDims(1); @@ -1279,6 +1360,14 @@ TEST_F(OpTest, ConcatOffset) { }); } +TEST_F(OpTest, Conj) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Conj") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, Conv2D) { Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); @@ -1293,11 +1382,12 @@ TEST_F(OpTest, Conv2D) { std::vector kernel_dims = {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out}; + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv2D") - .RandomInput(DT_FLOAT, data_dims) - .RandomInput(DT_FLOAT, kernel_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, data_dims) + .RandomInput(type, kernel_dims) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); @@ -1317,12 +1407,13 @@ TEST_F(OpTest, Conv2DBackpropFilter) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); Tensor kernel_shape = test::AsTensor(AsInt32s( {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out})); + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv2DBackpropFilter") - .RandomInput(DT_FLOAT, activations) + .RandomInput(type, activations) .Input(kernel_shape) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); @@ -1342,12 +1433,13 @@ TEST_F(OpTest, Conv2DBackpropInput) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], features_in, features_out}; + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv2DBackpropInput") .Input(in_shape) - .RandomInput(DT_FLOAT, kernel) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, kernel) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); @@ -1365,11 +1457,12 @@ TEST_F(OpTest, Conv3D) { std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out}; + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3D") - .RandomInput(DT_FLOAT, data) - .RandomInput(DT_FLOAT, kernel) - .Attr("T", DT_FLOAT) + .RandomInput(type, data) + .RandomInput(type, kernel) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); @@ -1389,12 +1482,13 @@ TEST_F(OpTest, Conv3DBackpropFilter) { Tensor kernel_shape = test::AsTensor( AsInt32s({d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out})); + DataType type = DT_FLOAT; // TODO(b/65408531): COMPLEX_64 support return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3DBackpropFilterV2") - .RandomInput(DT_FLOAT, activations) + .RandomInput(type, activations) .Input(kernel_shape) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); @@ -1413,17 +1507,34 @@ TEST_F(OpTest, Conv3DBackpropInput) { ImageDims(FORMAT_NHWC, batch, features_out, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], d.kernel_dims[2], features_in, features_out}; + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Conv3DBackpropInputV2") .Input(in_shape) - .RandomInput(DT_FLOAT, kernel) - .RandomInput(DT_FLOAT, backprop) - .Attr("T", DT_FLOAT) + .RandomInput(type, kernel) + .RandomInput(type, backprop) + .Attr("T", type) .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); } +TEST_F(OpTest, Cos) { + Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("Cos").RandomInput(type).Attr("T", type)); + }); +} + +TEST_F(OpTest, Cosh) { + Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); + return ExpectTfAndXlaOutputsAreClose( + OpTestBuilder("Cosh").RandomInput(type).Attr("T", type)); + }); +} + TEST_F(OpTest, DepthToSpace) { Repeatedly([this]() { int64 block = RandomDim(2, 5); @@ -1431,9 +1542,10 @@ TEST_F(OpTest, DepthToSpace) { input_dims[1] = (input_dims[1] + (block - 1)) / block; input_dims[2] = (input_dims[2] + (block - 1)) / block; input_dims[3] *= block * block; + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("DepthToSpace") - .RandomInput(DT_FLOAT, input_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, input_dims) + .Attr("T", type) .Attr("block_size", block)); }); } @@ -1449,12 +1561,14 @@ TEST_F(OpTest, DepthwiseConv2DNative) { std::vector kernel_dims = {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier}; + std::vector strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims); + strides[2] = strides[1]; // Current impl only supports equal strides return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("DepthwiseConv2dNative") .RandomInput(DT_FLOAT, input_dims) .RandomInput(DT_FLOAT, kernel_dims) .Attr("T", DT_FLOAT) - .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) + .Attr("strides", strides) .Attr("padding", d.padding == SAME ? "SAME" : "VALID")); }); } @@ -1472,32 +1586,20 @@ TEST_F(OpTest, DepthwiseConv2DBackpropFilter) { FORMAT_NHWC, batch, features_in * depth_multiplier, d.output_dims); Tensor kernel_shape = test::AsTensor(AsInt32s( {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier})); + std::vector strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims); + strides[2] = strides[1]; // Current impl only supports equal strides return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("DepthwiseConv2dNativeBackpropFilter") .RandomInput(DT_FLOAT, activations) .Input(kernel_shape) .RandomInput(DT_FLOAT, backprop) .Attr("T", DT_FLOAT) - .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) + .Attr("strides", strides) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); }); } -TEST_F(OpTest, Cos) { - Repeatedly([this]() { - return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Cos").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); - }); -} - -TEST_F(OpTest, Cosh) { - Repeatedly([this]() { - return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Cosh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); - }); -} - TEST_F(OpTest, DepthwiseConv2DBackpropInput) { Repeatedly([this]() { WindowedSpatialDims d = ChooseWindowedSpatialDims(2); @@ -1511,13 +1613,15 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) { FORMAT_NHWC, batch, features_in * depth_multiplier, d.output_dims); std::vector kernel = {d.kernel_dims[0], d.kernel_dims[1], features_in, depth_multiplier}; + std::vector strides = ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims); + strides[2] = strides[1]; // Current impl only supports equal strides return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("DepthwiseConv2dNativeBackpropInput") .Input(in_shape) .RandomInput(DT_FLOAT, kernel) .RandomInput(DT_FLOAT, backprop) .Attr("T", DT_FLOAT) - .Attr("strides", ImageDims(FORMAT_NHWC, 1, 1, d.stride_dims)) + .Attr("strides", strides) .Attr("padding", d.padding == SAME ? "SAME" : "VALID") .Attr("data_format", "NHWC")); }); @@ -1525,7 +1629,7 @@ TEST_F(OpTest, DepthwiseConv2DBackpropInput) { TEST_F(OpTest, Diag) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose(kAllXlaTypes); std::vector dims; // Diag causes a quadratic blowup in output size. int64 size; @@ -1540,7 +1644,7 @@ TEST_F(OpTest, Diag) { TEST_F(OpTest, DiagPart) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose(kAllXlaTypes); auto dims = RandomDims(1, 3); // Duplicate the random dims. std::vector doubled_dims(dims.size() * 2); @@ -1554,7 +1658,7 @@ TEST_F(OpTest, DiagPart) { TEST_F(OpTest, Div) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Div") .RandomInput(type, dims.first) @@ -1565,7 +1669,7 @@ TEST_F(OpTest, Div) { TEST_F(OpTest, DynamicStitch) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(2, 5)(generator()); OpTestBuilder builder("DynamicStitch"); builder.Attr("T", type); @@ -1650,7 +1754,7 @@ TEST_F(OpTest, SeluGrad) { TEST_F(OpTest, Equal) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Equal") .RandomInput(type, dims.first) @@ -1661,21 +1765,23 @@ TEST_F(OpTest, Equal) { TEST_F(OpTest, Exp) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Exp").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Exp").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Expm1) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Expm1").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Expm1").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, ExpandDims) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector in_dims = RandomDims(); Tensor dim(DT_INT32, TensorShape()); std::uniform_int_distribution d(-1 - in_dims.size(), in_dims.size()); @@ -1689,7 +1795,7 @@ TEST_F(OpTest, ExpandDims) { TEST_F(OpTest, Fill) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(); std::vector shape(dims.begin(), dims.end()); return ExpectTfAndXlaOutputsAreClose( @@ -1720,7 +1826,7 @@ TEST_F(OpTest, FloorDiv) { TEST_F(OpTest, FloorMod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("FloorMod") .RandomInput(type, dims.first) @@ -1731,7 +1837,7 @@ TEST_F(OpTest, FloorMod) { TEST_F(OpTest, Greater) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Greater") .RandomInput(type, dims.first) @@ -1742,7 +1848,7 @@ TEST_F(OpTest, Greater) { TEST_F(OpTest, GreaterEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("GreaterEqual") .RandomInput(type, dims.first) @@ -1751,6 +1857,14 @@ TEST_F(OpTest, GreaterEqual) { }); } +TEST_F(OpTest, Imag) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Imag") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, Invert) { Repeatedly([this]() { DataType type = DT_INT32; @@ -1769,7 +1883,7 @@ TEST_F(OpTest, L2Loss) { TEST_F(OpTest, Less) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Less") .RandomInput(type, dims.first) @@ -1780,7 +1894,7 @@ TEST_F(OpTest, Less) { TEST_F(OpTest, LessEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("LessEqual") .RandomInput(type, dims.first) @@ -1796,7 +1910,7 @@ TEST_F(OpTest, LinSpace) { return test::AsScalar(x); }; std::uniform_int_distribution distribution(-50, 50); - DataType type = Choose({DT_INT32, DT_INT64}); + auto type = Choose({DT_INT32, DT_INT64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("LinSpace") .RandomInput(DT_FLOAT, {}) @@ -1809,15 +1923,17 @@ TEST_F(OpTest, LinSpace) { TEST_F(OpTest, Log) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Log").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Log").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Log1p) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Log1p").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Log1p").RandomInput(type).Attr("T", DT_FLOAT)); }); } @@ -1914,10 +2030,11 @@ TEST_F(OpTest, MatMul) { std::swap(b_dims[0], b_dims[1]); } + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatMul") - .RandomInput(DT_FLOAT, a_dims) - .RandomInput(DT_FLOAT, b_dims) - .Attr("T", DT_FLOAT) + .RandomInput(type, a_dims) + .RandomInput(type, b_dims) + .Attr("T", type) .Attr("transpose_a", transpose_a) .Attr("transpose_b", transpose_b)); }); @@ -1925,7 +2042,7 @@ TEST_F(OpTest, MatMul) { TEST_F(OpTest, MatrixDiag) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiag") .RandomInput(type, RandomDims(1)) .Attr("T", type)); @@ -1934,7 +2051,7 @@ TEST_F(OpTest, MatrixDiag) { TEST_F(OpTest, MatrixDiagPart) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("MatrixDiagPart") .RandomInput(type, RandomDims(2)) .Attr("T", type)); @@ -1943,7 +2060,7 @@ TEST_F(OpTest, MatrixDiagPart) { TEST_F(OpTest, Max) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -1957,7 +2074,7 @@ TEST_F(OpTest, Max) { TEST_F(OpTest, Maximum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Maximum") .RandomInput(type, dims.first) @@ -2025,7 +2142,7 @@ TEST_F(OpTest, MaxPool3D) { TEST_F(OpTest, Mean) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); // TODO(phawkins): CPU and XLA differ output for reducing across a // size-0 dimension (nan vs 0). For now, require size >= 1. std::vector data_dims = RandomDims(0, kDefaultMaxRank, 1); @@ -2041,7 +2158,7 @@ TEST_F(OpTest, Mean) { TEST_F(OpTest, Min) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2055,7 +2172,7 @@ TEST_F(OpTest, Min) { TEST_F(OpTest, Minimum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Minimum") .RandomInput(type, dims.first) @@ -2076,7 +2193,7 @@ TEST_F(OpTest, Mod) { TEST_F(OpTest, Mul) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Mul") .RandomInput(type, dims.first) @@ -2087,7 +2204,7 @@ TEST_F(OpTest, Mul) { TEST_F(OpTest, Neg) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Neg").RandomInput(type).Attr("T", type)); }); @@ -2095,7 +2212,7 @@ TEST_F(OpTest, Neg) { TEST_F(OpTest, NotEqual) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("NotEqual") .RandomInput(type, dims.first) @@ -2106,7 +2223,7 @@ TEST_F(OpTest, NotEqual) { TEST_F(OpTest, OneHot) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(); int num_dims = dims.size(); @@ -2136,7 +2253,7 @@ TEST_F(OpTest, OneHot) { TEST_F(OpTest, OnesLike) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("OnesLike").RandomInput(type).Attr("T", type)); }); @@ -2144,7 +2261,7 @@ TEST_F(OpTest, OnesLike) { TEST_F(OpTest, Pack) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(1, 5)(generator()); std::vector dims = RandomDims(); @@ -2166,7 +2283,7 @@ TEST_F(OpTest, Pack) { // TODO(b/31741898): crashes on GPU. TEST_F(OpTest, Pad) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector t_dims = RandomDims(); // TODO(b/31741996): re-enable DT_INT64 when bug is fixed. @@ -2195,16 +2312,17 @@ TEST_F(OpTest, Pow) { // nontermination. Repeatedly([this]() { auto dims = BroadcastableDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Pow") - .RandomInput(DT_FLOAT, dims.first) - .RandomInput(DT_FLOAT, dims.second) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims.first) + .RandomInput(type, dims.second) + .Attr("T", type)); }); } TEST_F(OpTest, Prod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2238,15 +2356,23 @@ TEST_F(OpTest, Range) { TEST_F(OpTest, Rank) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Rank").RandomInput(type).Attr("T", type)); }); } +TEST_F(OpTest, Real) { + Repeatedly([this]() { + return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Real") + .RandomInput(DT_COMPLEX64) + .Attr("T", DT_COMPLEX64)); + }); +} + TEST_F(OpTest, RealDiv) { Repeatedly([this]() { - DataType type = DT_FLOAT; + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RealDiv") .RandomInput(type, dims.first) @@ -2257,18 +2383,20 @@ TEST_F(OpTest, RealDiv) { TEST_F(OpTest, Reciprocal) { Repeatedly([this]() { + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Reciprocal").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Reciprocal").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, ReciprocalGrad) { Repeatedly([this]() { std::vector dims = RandomDims(); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReciprocalGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } TEST_F(OpTest, Relu) { @@ -2307,7 +2435,7 @@ TEST_F(OpTest, ReluGrad) { TEST_F(OpTest, Reshape) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(); std::bernoulli_distribution random_bool; std::vector dims_before, dims_after; @@ -2335,24 +2463,24 @@ TEST_F(OpTest, Reshape) { TEST_F(OpTest, Reverse) { Repeatedly([this]() { std::vector dims = RandomDims(1); - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose(kAllXlaTypes); int64 rank = dims.size(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Reverse") .RandomInput(type, dims) .RandomInput(DT_BOOL, {rank}) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } TEST_F(OpTest, ReverseV2) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("ReverseV2") .RandomInput(type, data_dims) .Input(indices) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } @@ -2372,24 +2500,26 @@ TEST_F(OpTest, Round) { TEST_F(OpTest, Rsqrt) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Rsqrt").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Rsqrt").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, RsqrtGrad) { Repeatedly([this]() { auto dims = RandomDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("RsqrtGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } TEST_F(OpTest, Shape) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Shape").RandomInput(type).Attr("T", type)); }); @@ -2397,7 +2527,7 @@ TEST_F(OpTest, Shape) { TEST_F(OpTest, ShapeN) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); int n = std::uniform_int_distribution(1, 5)(generator()); OpTestBuilder builder("ShapeN"); builder.Attr("T", type); @@ -2411,24 +2541,26 @@ TEST_F(OpTest, ShapeN) { TEST_F(OpTest, Sigmoid) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sigmoid").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sigmoid").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, SigmoidGrad) { Repeatedly([this]() { auto dims = RandomDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SigmoidGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } TEST_F(OpTest, Sign) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Sign").RandomInput(type).Attr("T", type)); }); @@ -2436,21 +2568,23 @@ TEST_F(OpTest, Sign) { TEST_F(OpTest, Sin) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sin").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sin").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Sinh) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sinh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sinh").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Size) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Size").RandomInput(type).Attr("T", type)); }); @@ -2458,7 +2592,7 @@ TEST_F(OpTest, Size) { TEST_F(OpTest, Slice) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); std::vector begin(data_dims.size()), size(data_dims.size()); @@ -2562,10 +2696,11 @@ TEST_F(OpTest, SpaceToBatch) { CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals), TensorShape({num_block_dims, 2}))); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SpaceToBatch") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(paddings) - .Attr("T", DT_FLOAT) + .Attr("T", type) .Attr("block_size", block_size)); }); } @@ -2603,13 +2738,14 @@ TEST_F(OpTest, SpaceToBatchND) { CHECK(paddings.CopyFrom(AsIntTensor(DT_INT32, padding_vals), TensorShape({num_block_dims, 2}))); + auto type = Choose(kAllXlaTypes); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("SpaceToBatchND") - .RandomInput(DT_FLOAT, input_dims) + .RandomInput(type, input_dims) .Input(test::AsTensor( std::vector(block_dims.begin(), block_dims.end()))) .Input(paddings) - .Attr("T", DT_FLOAT)); + .Attr("T", type)); }); } @@ -2679,7 +2815,7 @@ TEST_F(OpTest, SparseSoftmaxCrossEntropyWithLogits) { TEST_F(OpTest, Split) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector dims = RandomDims(1); std::uniform_int_distribution ud; int32 dim = std::uniform_int_distribution( @@ -2699,18 +2835,20 @@ TEST_F(OpTest, Split) { TEST_F(OpTest, Sqrt) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Sqrt").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Sqrt").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, SqrtGrad) { Repeatedly([this]() { auto dims = RandomDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("SqrtGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } @@ -2726,7 +2864,7 @@ TEST_F(OpTest, SquaredDifference) { TEST_F(OpTest, Square) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("Square").RandomInput(type).Attr("T", type)); }); @@ -2734,7 +2872,7 @@ TEST_F(OpTest, Square) { TEST_F(OpTest, Squeeze) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector t_dims = RandomDims(0, kDefaultMaxRank, 0, 5); std::bernoulli_distribution random_bool; std::vector squeeze_dims; @@ -2752,7 +2890,7 @@ TEST_F(OpTest, Squeeze) { TEST_F(OpTest, Sub) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("Sub") .RandomInput(type, dims.first) @@ -2763,7 +2901,7 @@ TEST_F(OpTest, Sub) { TEST_F(OpTest, Sum) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); std::vector data_dims = RandomDims(); Tensor indices = RandomReductionIndices(data_dims.size()); bool keep_dims = Choose({false, true}); @@ -2777,7 +2915,7 @@ TEST_F(OpTest, Sum) { TEST_F(OpTest, StridedSlice) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); std::vector begin(data_dims.size()), end(data_dims.size()); std::vector strides(data_dims.size()); @@ -2822,7 +2960,7 @@ TEST_F(OpTest, StridedSlice) { TEST_F(OpTest, StridedSliceGrad) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); // Dimensions of the forward input. std::vector dims = RandomDims(); @@ -2875,31 +3013,34 @@ TEST_F(OpTest, StridedSliceGrad) { TEST_F(OpTest, Tan) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Tan").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Tan").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, Tanh) { Repeatedly([this]() { + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( - OpTestBuilder("Tanh").RandomInput(DT_FLOAT).Attr("T", DT_FLOAT)); + OpTestBuilder("Tanh").RandomInput(type).Attr("T", type)); }); } TEST_F(OpTest, TanhGrad) { Repeatedly([this]() { auto dims = RandomDims(); + auto type = Choose({DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TanhGrad") - .RandomInput(DT_FLOAT, dims) - .RandomInput(DT_FLOAT, dims) - .Attr("T", DT_FLOAT)); + .RandomInput(type, dims) + .RandomInput(type, dims) + .Attr("T", type)); }); } TEST_F(OpTest, Tile) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector t_dims = RandomDims(1); std::vector multiples(t_dims.size()); for (int i = 0; i < t_dims.size(); ++i) { @@ -2915,7 +3056,7 @@ TEST_F(OpTest, Tile) { TEST_F(OpTest, Transpose) { Repeatedly([this]() { - DataType type = Choose(kAllXlaTypes); + auto type = Choose(kAllXlaTypes); std::vector data_dims = RandomDims(); std::vector perm(data_dims.size()); std::iota(perm.begin(), perm.end(), 0); @@ -2940,7 +3081,7 @@ TEST_F(OpTest, TruncateDiv) { TEST_F(OpTest, TruncateMod) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT}); auto dims = BroadcastableDims(); return ExpectTfAndXlaOutputsAreClose(OpTestBuilder("TruncateMod") .RandomInput(type, dims.first) @@ -2951,7 +3092,7 @@ TEST_F(OpTest, TruncateMod) { TEST_F(OpTest, ZerosLike) { Repeatedly([this]() { - DataType type = Choose({DT_INT32, DT_FLOAT}); + auto type = Choose({DT_INT32, DT_FLOAT, DT_COMPLEX64}); return ExpectTfAndXlaOutputsAreClose( OpTestBuilder("ZerosLike").RandomInput(type).Attr("T", type)); }); diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index 71221b284d5b7ff4e3c259cafef9166dc2ef246c..76644380bdf2e0c24f6d363ddfaabdff836495d7 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -328,6 +328,131 @@ class UnaryOpsTest(XLATestCase): np.array([-1, -0.5, 0, 0.3], dtype=dtype), expected=np.array([-1, -64.0 / 127, 0, 38.0 / 127], dtype=dtype)) + def testComplexOps(self): + for dtype in self.complex_types: + # TODO(b/65408531): math_ops.acosh (needs pow) + # TODO(b/65408531): math_ops.asinh (needs pow) + + # TODO(b/65408531): Wider support for log (needs atan2). + atan2_supported = self.device == "XLA_GPU" + if atan2_supported: + self._assertOpOutputMatchesExpected( + math_ops.atanh, + np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype), + expected=np.arctanh( + np.array([0.1, 0.2j, 0.3 - 0.1j, 0.4 + 0.5j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.cosh, + np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype), + expected=np.cosh(np.array([1j, 2 - 3j, 3, 4 + 2j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.sinh, + np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype), + expected=np.sinh(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.exp, + np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype), + expected=np.exp(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.expm1, + np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype), + expected=np.expm1(np.array([[-1 + 2j, 3j, 2 - 3j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.reciprocal, + np.array([[1, 2j, 2 + 3j]], dtype=dtype), + expected=1.0 / np.array([[1, 2j, 2 + 3j]], dtype=dtype)) + + if atan2_supported: + self._assertOpOutputMatchesExpected( + math_ops.log, + np.array([[5j, 3 - 2j]], dtype=dtype), + expected=np.log(np.array([[5j, 3 - 2j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.sin, + np.array([[5j, 3 - 2j]], dtype=dtype), + expected=np.sin(np.array([[5j, 3 - 2j]], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.cos, + np.array([[5j, 3 - 2j]], dtype=dtype), + expected=np.cos(np.array([[5j, 3 - 2j]], dtype=dtype))) + + # TODO(b/34703906): improve log1p implementation and make tolerance + # tighter. + if atan2_supported: # TODO(b/34703906): log support + self._assertOpOutputMatchesExpected( + math_ops.log1p, + np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype), + expected=np.log1p( + np.array([[1e-14, 1e-15j, 0.6 - 0.3j]], dtype=dtype))) + + # TODO(b/34703906): math_ops.rsqrt (needs pow) + + # TODO(b/34703906): math_ops.sigmoid (needs tanh) + + # TODO(b/34703906): math_ops.sqrt (needs pow) + + self._assertOpOutputMatchesExpected( + math_ops.tan, + np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype), + expected=np.tan(np.array([1, 2j, 2 - 3j, 4 + 5j], dtype=dtype))) + + # TODO(b/34703906): math_ops.tanh (as itself) + + ctypes = {np.complex64: np.float32} + self._assertOpOutputMatchesExpected( + math_ops.abs, + np.array([[3 - 4j, -1j, np.inf]], dtype=dtype), + expected=np.array([[5, 1, np.inf]], dtype=ctypes[dtype])) + + self._assertOpOutputMatchesExpected( + math_ops.negative, + np.array([[-1 + 2j, -3j]], dtype=dtype), + expected=np.array([[1 - 2j, 3j]], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + math_ops.square, + np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype), + expected=np.array([[-2 - 3j, 3 + 4j, 5j]], dtype=dtype)**2) + + self._assertOpOutputMatchesExpected( + array_ops.zeros_like, + np.array([[4j, 3 - 2j], [2, -1j]], dtype=dtype), + expected=np.array([[0, 0], [0, 0]], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + array_ops.ones_like, + np.array([[-4j, 3 + 2j], [2, -1j]], dtype=dtype), + expected=np.array([[1, 1], [1, 1]], dtype=dtype)) + + if atan2_supported: # TODO(b/34703906): atan2 support + self._assertOpOutputMatchesExpected( + math_ops.angle, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.angle( + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype))) + + self._assertOpOutputMatchesExpected( + math_ops.conj, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.array([1 - 3j, -4 - 7j, 2.7, 3j], dtype=dtype)) + + self._assertOpOutputMatchesExpected( + math_ops.imag, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.array([3, 7, 0, -3], dtype=ctypes[dtype])) + + self._assertOpOutputMatchesExpected( + math_ops.real, + np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), + expected=np.array([1, -4, 2.7, 0], dtype=ctypes[dtype])) + def testIntOps(self): for dtype in self.int_types: self._assertOpOutputMatchesExpected( @@ -399,11 +524,14 @@ class UnaryOpsTest(XLATestCase): def testCast(self): shapes = [[], [4], [2, 3], [2, 0, 4]] - types = [dtypes.bool, dtypes.int32, dtypes.float32] + types = [dtypes.bool, dtypes.int32, dtypes.float32] + self.complex_tf_types for shape in shapes: for src_type in types: for dst_type in types: src = np.arange(np.prod(shape)).astype(src_type.as_numpy_dtype) + if src_type in self.complex_tf_types: + src += (np.arange(np.prod(shape)) * 2j).astype( + src_type.as_numpy_dtype) src = src.reshape(shape) dst = src.astype(dst_type.as_numpy_dtype) diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py index fdf3f9fb6ada762751f8639af29bec0b0d9a8b01..c50342dee45eba6ae54f01653ecc81ef096b547b 100644 --- a/tensorflow/compiler/tests/variable_ops_test.py +++ b/tensorflow/compiler/tests/variable_ops_test.py @@ -43,7 +43,7 @@ class VariableOpsTest(XLATestCase): # Regression test for a bug where computations with one non-constant # output and one variable update were mishandled. for dtype in self.numeric_types: - init = np.array([[1, 2], [3, 4]], dtype=dtype) + init = np.array([[1, 2j], [3, 4]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) @@ -51,82 +51,91 @@ class VariableOpsTest(XLATestCase): x = v.assign_add(p) with ops.control_dependencies([x]): y = v.read_value() - self.assertAllClose(np.array([[2, 3], [4, 5]], dtype=dtype), - sess.run(y, {p: 1})) + self.assertAllClose( + np.array([[2, 1 + 2j], [4, 5]]).astype(dtype), sess.run(y, { + p: 1 + })) def testSparseRead0DIndices(self): for dtype in self.numeric_types: - init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) + init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8j, 9, 10, + 11]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read(2) - self.assertAllClose(np.array([8, 9, 10, 11], dtype=dtype), sess.run(x)) + self.assertAllClose( + np.array([8j, 9, 10, 11]).astype(dtype), sess.run(x)) def testSparseRead1DIndices(self): for dtype in self.numeric_types: - init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) + init = np.array([[0, 1, 2, 3], [4, 5, 6j, 7], [8, 9, 10, + 11]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([2, 1]) self.assertAllClose( - np.array([[8, 9, 10, 11], [4, 5, 6, 7]], dtype=dtype), sess.run(x)) + np.array([[8, 9, 10, 11], [4, 5, 6j, 7]]).astype(dtype), + sess.run(x)) def testSparseRead2DIndices(self): for dtype in self.numeric_types: - init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) + init = np.array([[0, 1, 2j, 3], [4, 5, 6, 7], [8, 9, 10, + 11]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([[2, 1], [0, 2]]) self.assertAllClose( - np.array( - [[[8, 9, 10, 11], [4, 5, 6, 7]], [[0, 1, 2, 3], [8, 9, 10, - 11]]], - dtype=dtype), sess.run(x)) + np.array([[[8, 9, 10, 11], [4, 5, 6, 7]], + [[0, 1, 2j, 3], [8, 9, 10, 11]]]).astype(dtype), + sess.run(x)) def testSparseRead2DIndices3DTensor(self): for dtype in self.numeric_types: - init = np.array( - [[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]], - [[20, 21, 22], [23, 24, 25]], [[30, 31, 32], [33, 34, 35]]], - dtype=dtype) + init = np.array([[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]], + [[20, 21, 22], [23, 24j, 25]], + [[30, 31, 32], [33, 34, 35]]]).astype(dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([[2, 1], [3, 0]]) self.assertAllClose( np.array( - [[[[20, 21, 22], [23, 24, 25]], [[10, 11, 12], [13, 14, 15]]], + [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]], [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]]], - dtype=dtype), sess.run(x)) + ).astype(dtype), sess.run(x)) def testReadWrite(self): """Tests initialization, reading, and writing a resource variable.""" - with self.test_session() as session: - with self.test_scope(): - with variable_scope.variable_scope("ascope", use_resource=True): - x = variable_scope.get_variable( - "x", - shape=[], - dtype=dtypes.float32, - initializer=init_ops.constant_initializer(2)) - a = x.read_value() - with ops.control_dependencies([a]): - b = state_ops.assign(x, 47) - with ops.control_dependencies([b]): - c = x.read_value() - with ops.control_dependencies([c]): - d = state_ops.assign_add(x, 3) - with ops.control_dependencies([d]): - e = x.read_value() - - session.run(variables.global_variables_initializer()) - v1, v2, v3 = session.run([a, c, e]) - self.assertAllClose(2.0, v1) - self.assertAllClose(47.0, v2) - self.assertAllClose(50.0, v3) + for dtype in self.numeric_types: + with self.test_session() as session: + print(ops.get_default_graph()) + with self.test_scope(): + with variable_scope.variable_scope("ascope", use_resource=True): + x = variable_scope.get_variable( + "x", + shape=[], + dtype=dtype, + initializer=init_ops.constant_initializer(2)) + a = x.read_value() + with ops.control_dependencies([a]): + b = state_ops.assign(x, dtype(47)) + with ops.control_dependencies([b]): + c = x.read_value() + with ops.control_dependencies([c]): + d = state_ops.assign_add(x, np.array(6 + 2j).astype(dtype)) + with ops.control_dependencies([d]): + e = state_ops.assign_sub(x, dtype(3)) + with ops.control_dependencies([e]): + f = x.read_value() + + session.run(variables.global_variables_initializer()) + v1, v2, v3 = session.run([a, c, f]) + self.assertAllClose(dtype(2), v1) + self.assertAllClose(dtype(47), v2) + self.assertAllClose(np.array(50 + 2j).astype(dtype), v3) def testTraining(self): """Tests a gradient descent step for a simple model.""" diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py index da6dc88f1fb07200799f8ee231fc04628b265e24..0be127997e5211f810ca791187486760881fe172 100644 --- a/tensorflow/compiler/tests/xla_test.py +++ b/tensorflow/compiler/tests/xla_test.py @@ -63,12 +63,19 @@ class XLATestCase(test.TestCase): self.float_tf_types = [ dtype for dtype in self.all_tf_types if dtype.is_floating ] - self.numeric_tf_types = self.int_tf_types + self.float_tf_types + self.complex_tf_types = [ + dtype for dtype in self.all_tf_types if dtype.is_complex + ] + self.numeric_tf_types = ( + self.int_tf_types + self.float_tf_types + self.complex_tf_types) self.all_types = [dtype.as_numpy_dtype for dtype in self.all_tf_types] self.int_types = [dtype.as_numpy_dtype for dtype in self.int_tf_types] self.float_types = [dtype.as_numpy_dtype for dtype in self.float_tf_types] - self.numeric_types = self.int_types + self.float_types + self.complex_types = [ + dtype.as_numpy_dtype for dtype in self.complex_tf_types + ] + self.numeric_types = self.int_types + self.float_types + self.complex_types # Parse the manifest file, if any, into a regex identifying tests to # disable diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index 7865f16e5381b95c800eaf783a9b22ffd625c40c..3c94bcafc1d19b1bc54887e6f2c25b1886be646e 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -87,6 +87,7 @@ cc_library( "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:client_library", "//tensorflow/compiler/xla/client:local_client", + "//tensorflow/compiler/xla/service:cpu_plugin", "//tensorflow/compiler/xla/service/cpu:cpu_executable", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", @@ -227,7 +228,6 @@ tf_cc_test( "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:local_client", - "//tensorflow/compiler/xla/service:cpu_plugin", "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", diff --git a/tensorflow/compiler/tf2xla/const_analysis.cc b/tensorflow/compiler/tf2xla/const_analysis.cc index bf75f85db041087d8770bd21494f8e1a7fe8c1b5..102a2cf07b51486bb445b0311966717b7e82ace6 100644 --- a/tensorflow/compiler/tf2xla/const_analysis.cc +++ b/tensorflow/compiler/tf2xla/const_analysis.cc @@ -67,6 +67,7 @@ Status BackwardsConstAnalysis(const Graph& g, {"Min", "reduction_indices"}, {"OneHot", "depth"}, {"Pad", "paddings"}, + {"PadV2", "paddings"}, {"MirrorPad", "paddings"}, {"Prod", "reduction_indices"}, {"RandomStandardNormal", "shape"}, diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc index 40bc164c50a1666344fec37affa47f8c605496cb..35b6960a98cda1bf098f3e01cac3df8173bdc729 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow.cc @@ -74,6 +74,18 @@ struct Frame { std::unordered_set nodes; }; +// Returns a textual representation of the names of the nodes in the input. +template +string NodesToString(const T& nodes) { + return strings::StrCat("{", + str_util::Join(nodes, ",", + [](string* output, const Node* node) { + strings::StrAppend(output, + node->name()); + }), + "}"); +} + // Copies a subgraph from `graph` to `output` by performing a reverse DFS // starting at nodes in vector `stack`. // `node_map` is a vector indexed by source node ID to dest nodes. @@ -93,12 +105,13 @@ Status CopySubgraph(const Graph& graph, const Frame* frame, std::vector stack, const std::vector& squash_src_outputs, std::vector* node_map, Graph* output) { + VLOG(3) << "Stack: " << NodesToString(stack); std::vector visited(graph.num_node_ids(), false); while (!stack.empty()) { Node* n = stack.back(); stack.pop_back(); - VLOG(3) << "Copying node " << n->name(); + VLOG(5) << "Copying node " << n->name(); if (visited[n->id()]) continue; visited[n->id()] = true; @@ -577,8 +590,13 @@ class FunctionalizeCond { // id in the original graph. struct CondArgs { struct CondCmp { - bool operator()(const Node* a, const Node* b) { - return a->id() < b->id(); + bool operator()(const Node* lhs, const Node* rhs) const { + bool lhs_is_resource = + lhs->num_inputs() > 0 ? (lhs->input_type(0) == DT_RESOURCE) : false; + bool rhs_is_resource = + rhs->num_inputs() > 0 ? (rhs->input_type(0) == DT_RESOURCE) : false; + return std::tie(lhs_is_resource, lhs->name()) < + std::tie(rhs_is_resource, rhs->name()); } }; Node* conditional = nullptr; @@ -613,7 +631,10 @@ class FunctionalizeCond { // If `from` and `to` correspond to different clusters, then merge the nodes // in the clustered graph corresponding to `from` and `to`. - void ContractEdge(Cluster* from, Cluster* to); + // + // If `remove_from_graph` is specified then the `from` node is also removed + // from the clustered graph post contracting the edge. + void ContractEdge(Cluster* from, Cluster* to, bool remove_from_graph = false); // Converts a Merge node to a XlaIf. This encapsulates the process of // extracting the bodies needed for the then and else branch, creates a XlaIf @@ -621,6 +642,10 @@ class FunctionalizeCond { // merge node with a XlaIf. Status ConvertMergeToXlaIf(Cluster* merge_cluster); + // Removes a Switch cluster feeding directly into a Merge cluster by removing + // the Switch and Merge nodes and collapsing into a single cluster. + Status RemoveTrivialMerge(Cluster* merge_cluster); + // Returns the switch cluster corresponding to the merge node. This function // only returns the switch cluster in the simple case where we have a switch // node is the entry of a diamond corresponding to a conditional: @@ -629,7 +654,10 @@ class FunctionalizeCond { // / \ // Branch Branch // \ / - // merge_cluster + // merge_cluster + // + // Note: either of the branches may be empty. The case where both branches are + // empty is handled by RemoveTrivialMerge. gtl::optional GetSwitchCluster(const Cluster& merge_cluster); // Determines the arguments needed as input to the Merge cluster originating @@ -661,8 +689,8 @@ class FunctionalizeCond { template void RemoveUnusedArgs(const T& args); - // Removes all Merge nodes that are unused. - void RemoveUnusedMergeNodes(Cluster* merge_cluster); + // Removes all Merge nodes in merge_cluster. + void RemoveMergeNodes(Cluster* merge_cluster); // Returns the representative member of the corresponding cluster. ClusterHandle Representative(const Node* node) { @@ -687,7 +715,7 @@ std::ostream& operator<<(std::ostream& os, // between the nodes and the nodes in each cluster. string DebugString(const Graph& graph, FunctionalizeCond::ClusterHandle::Vector* clusters) { - string ret = "digraph {\ncompound=true;labeljust=\"r\";\n"; + string ret = "digraph {\ncompound=true;labeljust=\"r\";ranksep=0.24\n"; std::map subgraphs; for (Node* n : graph.nodes()) { if (n->IsOp()) { @@ -697,8 +725,8 @@ string DebugString(const Graph& graph, } for (auto kv : subgraphs) { strings::StrAppend(&ret, "subgraph cluster_", kv.first.ToString(), " {\n", - "label = \"", kv.first.ToString(), "\";\n", kv.second, - "}\n"); + "style=filled; color=lightgrey;", "label = \"", + kv.first.ToString(), "\";\n", kv.second, "}\n"); } for (Node* n : graph.nodes()) { if (!n->IsOp()) { @@ -713,6 +741,24 @@ string DebugString(const Graph& graph, return strings::StrCat(ret, "}"); } +string DebugString(const FunctionalizeCond::ClusteredGraph& clustered_graph) { + string ret = "digraph {\ncompound=true;labeljust=\"r\";\n"; + auto name = [](const FunctionalizeCond::Cluster& cluster) { + return cluster.representative.ToString(); + }; + for (auto kv : clustered_graph) { + strings::StrAppend(&ret, kv.first.ToString(), " [label=\"", name(kv.second), + " (", kv.second.switch_nodes.size(), ", ", + kv.second.merge_nodes.size(), ")\"];\n"); + } + for (auto kv : clustered_graph) { + for (auto in : kv.second.in_nodes) { + strings::StrAppend(&ret, name(*in), " -> ", name(kv.second), ";\n"); + } + } + return strings::StrCat(ret, "}"); +} + bool IsDeadSwitch(const Node* node) { for (const Edge* e : node->out_edges()) { const Node* dst = e->dst(); @@ -754,21 +800,22 @@ void FunctionalizeCond::CreateClusters() { // conservatively assuming all merge nodes become XlaIf nodes. clusters_.resize(clusters_.size() + merge_nodes_.size()); - // Merge a cluster with its input, unless the input is a Switch node or the - // node is a Merge node. + // Merge a cluster with its input, unless the input is a Switch node or + // the node is a Merge node. for (const Node* node : graph_->nodes()) { - if (IsMerge(node) || !node->IsOp()) { + if (IsMerge(node) || IsSwitch(node) || !node->IsOp()) { continue; } for (const Node* in : node->in_nodes()) { - if (!IsSwitch(in) && in->IsOp()) { + if (in->IsOp() && !IsSwitch(in) && !IsMerge(in)) { clusters_.at(node).Merge(&clusters_.at(in)); } } } } -void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) { +void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to, + bool remove_from_graph) { VLOG(3) << "ContractEdge from = " << from->representative << " to = " << to->representative; if (from->representative == to->representative) { @@ -801,6 +848,10 @@ void FunctionalizeCond::ContractEdge(Cluster* from, Cluster* to) { to->out_nodes.erase(from); clusters_.at(to->representative).Merge(&clusters_.at(from->representative)); from->visited = true; + + if (remove_from_graph) { + clustered_graph_.erase(from->representative); + } } void FunctionalizeCond::CreateClusteredGraph() { @@ -839,6 +890,22 @@ void FunctionalizeCond::CreateClusteredGraph() { update_cluster_for_node(node).merge_nodes.insert(node); } + // Merge Switch nodes with common predicate. + std::unordered_map> predicate_to_switch; + for (Node* node : switch_nodes_) { + Node* tmp; + TF_CHECK_OK(node->input_node(1, &tmp)); + predicate_to_switch[tmp].push_back(node); + } + for (auto kv : predicate_to_switch) { + Cluster& first = clustered_graph_.at(Representative(kv.second.front())); + for (Node* switch_node : kv.second) { + ClusterHandle handle = Representative(switch_node); + Cluster& cluster = clustered_graph_.at(handle); + ContractEdge(&cluster, &first, /*remove_from_graph=*/true); + } + } + // Merge Merge nodes with common input together. for (Node* node : merge_nodes_) { Cluster& cluster = clustered_graph_.at(Representative(node)); @@ -847,35 +914,47 @@ void FunctionalizeCond::CreateClusteredGraph() { continue; } Cluster& cluster_node_in = clustered_graph_.at(Representative(in)); + // ContractEdge can modify out_nodes of cluster_node_in, so traverse + // over out_nodes assuming it does. for (auto it = cluster_node_in.out_nodes.begin(); it != cluster_node_in.out_nodes.end();) { - ContractEdge(*it++, &cluster); + if (!(*it)->merge_nodes.empty()) { + ContractEdge(*it++, &cluster, /*remove_from_graph=*/true); + } else { + ++it; + } } } } - VLOG(3) << "ClusteredGraph: " << DebugString(*graph_, &clusters_); + VLOG(3) << "Graph with clusters: " << DebugString(*graph_, &clusters_); + VLOG(3) << "ClusteredGraph: " << DebugString(clustered_graph_); } gtl::optional FunctionalizeCond::GetSwitchCluster( const Cluster& merge_cluster) { VLOG(3) << "GetSwitchCluster for " << merge_cluster.representative; gtl::optional switch_cluster; - if (merge_cluster.in_nodes.size() != 2) { + if (merge_cluster.in_nodes.size() > 2) { return gtl::nullopt; } - for (const Cluster* in : merge_cluster.in_nodes) { - if (in->in_nodes.size() != 1) { + for (Cluster* in : merge_cluster.in_nodes) { + Cluster* cluster = in; + if (in->switch_nodes.empty()) { + if (in->in_nodes.size() != 1) { + return gtl::nullopt; + } + // There is only a single `in` cluster. + cluster = *in->in_nodes.begin(); + } + if (cluster->switch_nodes.empty()) { return gtl::nullopt; } - for (auto inin : in->in_nodes) { - if (switch_cluster.has_value()) { - if (*switch_cluster != inin) { - return gtl::nullopt; - } - } else { - switch_cluster = inin; - } + + if (switch_cluster.has_value() && *switch_cluster != cluster) { + return gtl::nullopt; + } else { + switch_cluster = cluster; } } return switch_cluster; @@ -889,6 +968,9 @@ xla::StatusOr FunctionalizeCond::DetermineCondArgs( auto feeds_into_branch_cluster = [&](Node* switch_cluster) { for (Node* out : switch_cluster->out_nodes()) { ClusterHandle repr = Representative(out); + if (repr == merge_cluster.representative) { + return true; + } for (Cluster* in : merge_cluster.in_nodes) { if (repr == in->representative) { return true; @@ -919,12 +1001,9 @@ xla::StatusOr FunctionalizeCond::DetermineCondArgs( xla::StatusOr FunctionalizeCond::BuildAndAddXlaIfOp( const CondArgs& cond_args, const Cluster& merge_cluster, const std::vector& outputs) { - VLOG(2) << "Build if op for {" - << str_util::Join(merge_cluster.merge_nodes, ", ", - [](string* out, const Node* node) { - strings::StrAppend(out, node->name()); - }) - << "}"; + VLOG(2) << "Build if op for " << NodesToString(merge_cluster.merge_nodes) + << " with input " << NodesToString(cond_args.args); + NodeDef if_def; // Create a new If node using the name of the merge node. NodeDefBuilder builder( @@ -941,6 +1020,7 @@ xla::StatusOr FunctionalizeCond::BuildAndAddXlaIfOp( auto body = xla::MakeUnique(graph_->op_registry()); TF_RETURN_IF_ERROR( ExtractBody(cond_args, merge_cluster, outputs, i, body.get())); + VLOG(3) << "Body " << branch[i] << ": " << DebugString(body.get()); FunctionDef body_fdef; TF_RETURN_IF_ERROR(GraphToFunctionDef(*body, body_name.name(), &body_fdef)); TF_RETURN_IF_ERROR(library_->AddFunctionDef(body_fdef)); @@ -1001,10 +1081,7 @@ void FunctionalizeCond::RemoveClusterNodes(Cluster* cluster) { template void FunctionalizeCond::RemoveUnusedArgs(const T& args) { - VLOG(2) << "RemoveUnusedArgs among: " - << str_util::Join(args, ", ", [](string* output, const Node* node) { - strings::StrAppend(output, node->name()); - }); + VLOG(2) << "RemoveUnusedArgs among: " << NodesToString(args); std::deque to_delete; for (Node* arg : args) { @@ -1029,7 +1106,8 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args, const Cluster& merge_cluster, const std::vector& outputs, int input_edge, Graph* body) { - VLOG(2) << "ExtractBody for " << merge_cluster.representative; + VLOG(2) << "ExtractBody for " << merge_cluster.representative + << " along edge " << input_edge; std::vector squash_src_outputs(graph_->num_node_ids(), false); std::vector node_map(graph_->num_node_ids(), nullptr); int arg_count = 0; @@ -1037,11 +1115,6 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args, DataType dtype = arg->input_type(0); TF_ASSIGN_OR_RETURN(Node * arg_node, BuildArgNode(body, dtype, arg_count++)); - if (dtype == DT_RESOURCE) { - bool constant; - TF_RETURN_IF_ERROR(GetNodeAttr(arg->attrs(), "is_constant", &constant)); - TF_RET_CHECK(constant); - } node_map.at(arg->id()) = arg_node; squash_src_outputs.at(arg->id()) = true; } @@ -1053,12 +1126,21 @@ Status FunctionalizeCond::ExtractBody(const CondArgs& cond_args, TF_ASSIGN_OR_RETURN(node_map.at(node->id()), BuildRetvalNode(body, node->output_type(0), /*index=*/j)); - Node* in; - TF_RETURN_IF_ERROR(node->input_node(input_edge, &in)); + const Edge* in_edge; + TF_RETURN_IF_ERROR(node->input_edge(input_edge, &in_edge)); + Node* in = in_edge->src(); if (node_map.at(in->id()) == nullptr) { node_map.at(in->id()) = body->CopyNode(in); } - body->AddEdge(node_map.at(in->id()), j, node_map.at(node->id()), 0); + + if (cond_args.args.find(in) == cond_args.args.end()) { + body->AddEdge(node_map.at(in->id()), in_edge->src_output(), + node_map.at(node->id()), 0); + } else { + body->AddEdge(node_map.at(in->id()), 0, node_map.at(node->id()), 0); + // Don't include input nodes that are already just returned in stack. + continue; + } stack.push_back(in); } @@ -1108,17 +1190,46 @@ Status FunctionalizeCond::AddOutputEdges(const std::vector& outputs, return Status::OK(); } -void FunctionalizeCond::RemoveUnusedMergeNodes(Cluster* merge_cluster) { - VLOG(3) << "RemoveUnusedMergeNodes for " << merge_cluster->representative; +void FunctionalizeCond::RemoveMergeNodes(Cluster* merge_cluster) { + VLOG(3) << "RemoveMergeNodes for " << merge_cluster->representative; // Remove all merge nodes now dead post extraction of If. for (auto it = merge_cluster->merge_nodes.begin(); it != merge_cluster->merge_nodes.end();) { Node* node = *it; - if (node->out_edges().empty()) { - graph_->RemoveNode(node); - merge_cluster->merge_nodes.erase(*it++); + graph_->RemoveNode(node); + merge_cluster->merge_nodes.erase(*it++); + } +} + +Status FunctionalizeCond::RemoveTrivialMerge(Cluster* merge_cluster) { + Cluster* switch_cluster = *merge_cluster->in_nodes.begin(); + if (switch_cluster->switch_nodes.empty()) { + return errors::FailedPrecondition( + "Not a trivial merge: no Switch node feeding into Merge node"); + } + + for (auto it = merge_cluster->merge_nodes.begin(); + it != merge_cluster->merge_nodes.end();) { + // We have the following structure: + // Op -> Switch -> Merge -> Consumer + // and we want to transform it to: + // Op -> Consumer + Node* merge_node = *it; + Node* switch_node; + const Edge* in = nullptr; + TF_RETURN_IF_ERROR(merge_node->input_node(0, &switch_node)); + TF_RETURN_IF_ERROR(switch_node->input_edge(0, &in)); + for (auto out : merge_node->out_edges()) { + int src_output = out->dst_input() == Graph::kControlSlot + ? Graph::kControlSlot + : in->src_output(); + graph_->AddEdge(in->src(), src_output, out->dst(), out->dst_input()); } + graph_->RemoveNode(*it++); } + RemoveUnusedArgs(switch_cluster->switch_nodes); + + return Status::OK(); } Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { @@ -1127,12 +1238,8 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { if (!switch_cluster.has_value()) { return errors::FailedPrecondition( "Merge cluster was not part of a simple conditional in the clustered " - "graph. Graph nodes in merge cluster {", - str_util::Join(merge_cluster->merge_nodes, ", ", - [](string* output, Node* node) { - strings::StrAppend(output, node->name()); - }), - "}"); + "graph. Graph nodes in merge cluster ", + NodesToString(merge_cluster->merge_nodes)); } TF_ASSIGN_OR_RETURN(auto cond_args, DetermineCondArgs(*merge_cluster, **switch_cluster)); @@ -1140,9 +1247,7 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { // Sort the outputs by ID to produce more stable output. std::vector outputs(merge_cluster->merge_nodes.begin(), merge_cluster->merge_nodes.end()); - std::sort( - outputs.begin(), outputs.end(), - [](const Node* lhs, const Node* rhs) { return lhs->id() < rhs->id(); }); + std::sort(outputs.begin(), outputs.end(), CondArgs::CondCmp()); // Extract bodies and builds a If operator. TF_ASSIGN_OR_RETURN(Node * if_node, @@ -1153,15 +1258,17 @@ Status FunctionalizeCond::ConvertMergeToXlaIf(Cluster* merge_cluster) { // Remove the old nodes from the graph_ and contract the edges of the // clustered graph. for (auto in : merge_cluster->in_nodes) { - RemoveClusterNodes(in); + if (in != *switch_cluster) { + RemoveClusterNodes(in); + } } + RemoveMergeNodes(merge_cluster); RemoveUnusedArgs(cond_args.args); auto in_nodes = merge_cluster->in_nodes; for (auto it = in_nodes.begin(); it != in_nodes.end();) { ContractEdge(*it++, merge_cluster); } ContractEdge(*switch_cluster, merge_cluster); - RemoveUnusedMergeNodes(merge_cluster); clusters_[if_node].Get() = ClusterHandle(merge_cluster->representative); return Status::OK(); @@ -1230,7 +1337,27 @@ Status FunctionalizeCond::Functionalize(Graph* graph, for (auto it = queue.begin(); it != queue.end();) { Cluster* merge_cluster = (*it).second; ++it; - TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster)); + if (merge_cluster->in_nodes.size() == 1) { + TF_RETURN_IF_ERROR(fc.RemoveTrivialMerge(merge_cluster)); + } else { + TF_RETURN_IF_ERROR(fc.ConvertMergeToXlaIf(merge_cluster)); + } + + // Contract newly Merge free merge_cluster with incoming nodes without + // Switch or Merge nodes. + std::vector in_nodes(merge_cluster->in_nodes.begin(), + merge_cluster->in_nodes.end()); + for (auto in : in_nodes) { + if (in->merge_nodes.empty() && in->switch_nodes.empty()) { + fc.ContractEdge(in, merge_cluster); + } + } + } + + if (!fc.switch_nodes_.empty()) { + return errors::Internal( + "Failed to functionalize control flow with Switch nodes remaining: ", + NodesToString(fc.switch_nodes_)); } return Status::OK(); } @@ -1241,7 +1368,7 @@ Status FunctionalizeCond::Functionalize(Graph* graph, // functional equivalents. Status FunctionalizeControlFlow(Graph* graph, FunctionLibraryDefinition* library) { - VLOG(2) << "FunctionalizeControlFlow: " + VLOG(2) << "FunctionalizeControlFlow (initial): " << dump_graph::DumpGraphToFile("functionalize_initial", *graph); // Note: BuildControlFlowInfo() requires that the graph's source node is // connected to all source nodes in the graph. Many graphs violate this @@ -1319,7 +1446,11 @@ Status FunctionalizeControlFlow(Graph* graph, // FunctionalizeControlFlow is invoked for every function, so the loops's // bodies and conditionals that were extracted into functions will be handled // in successive invocations. - return FunctionalizeCond::Functionalize(graph, library); + TF_RETURN_IF_ERROR(FunctionalizeCond::Functionalize(graph, library)); + + VLOG(2) << "FunctionalizeControlFlow (final): " + << dump_graph::DumpGraphToFile("functionalize_final", *graph); + return Status::OK(); } } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc index 4acdf1a26d60778a70452bc9857dfd181ca24be8..01d2b282751f387cfa9c8887cdeb48090c96bff4 100644 --- a/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc +++ b/tensorflow/compiler/tf2xla/functionalize_control_flow_test.cc @@ -110,7 +110,7 @@ TEST(FunctionalizeControlFlow, Conditional) { auto x = ops::Placeholder(scope.WithOpName("x"), DT_INT32); auto less = ops::Less(scope.WithOpName("cond/Less"), y, x); auto if_op = ops::XlaIf(scope.WithOpName("cond/Merge_If"), less, - std::initializer_list{x, y, less}, then_fn, + std::initializer_list{less, y, x}, then_fn, else_fn, {DT_INT32}); GraphDef expected; TF_EXPECT_OK(scope.ToGraphDef(&expected)); @@ -120,10 +120,10 @@ TEST(FunctionalizeControlFlow, Conditional) { // then body. { Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2); - auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_2); + auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto identity = ops::Identity(scope.WithOpName("cond/Identity"), arg_0); auto cond = ops::Const( scope.WithOpName("cond").WithControlDependencies(identity), 17); auto mul = ops::Mul(scope.WithOpName("cond/Mul"), arg_1, cond); @@ -136,20 +136,20 @@ TEST(FunctionalizeControlFlow, Conditional) { TF_EXPECT_OK(InstantiateFunctionForTest(then_fn.name(), library, &result)); EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types); + EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types); TF_EXPECT_GRAPH_EQ(expected, result.gdef); } // else body. { Scope scope = Scope::NewRootScope().ExitOnError(); - auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_INT32, 0); + auto arg_0 = ops::_Arg(scope.WithOpName("_arg0"), DT_BOOL, 0); auto arg_1 = ops::_Arg(scope.WithOpName("_arg1"), DT_INT32, 1); - auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_BOOL, 2); - auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_2); + auto arg_2 = ops::_Arg(scope.WithOpName("_arg2"), DT_INT32, 2); + auto identity = ops::Identity(scope.WithOpName("cond/Identity_1"), arg_0); auto cond_1 = ops::Const( scope.WithOpName("cond_1").WithControlDependencies(identity), 23); - auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_0, cond_1); + auto add = ops::Add(scope.WithOpName("cond/false/add"), arg_2, cond_1); auto retval0 = ops::_Retval(scope.WithOpName("_retval0_RetVal"), add, 0); GraphDef expected; @@ -159,7 +159,7 @@ TEST(FunctionalizeControlFlow, Conditional) { TF_EXPECT_OK(InstantiateFunctionForTest(else_fn.name(), library, &result)); EXPECT_EQ(DataTypeVector{DT_INT32}, result.ret_types); - EXPECT_EQ((DataTypeVector{DT_INT32, DT_INT32, DT_BOOL}), result.arg_types); + EXPECT_EQ((DataTypeVector{DT_BOOL, DT_INT32, DT_INT32}), result.arg_types); TF_EXPECT_GRAPH_EQ(expected, result.gdef); } } diff --git a/tensorflow/compiler/tf2xla/graph_compiler.cc b/tensorflow/compiler/tf2xla/graph_compiler.cc index 6f2f59d98fb03ffd7db19aaa70774ecfa4b78ce9..8062f0c03ca60e88bd5c021092dceb105232219f 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.cc +++ b/tensorflow/compiler/tf2xla/graph_compiler.cc @@ -38,6 +38,7 @@ limitations under the License. #include "tensorflow/core/graph/algorithm.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/public/version.h" @@ -84,9 +85,20 @@ Status PrepareArguments(XlaOpKernelContext* ctx, Graph* graph, } } // namespace Status GraphCompiler::Compile() { - std::vector bindings(graph_->num_node_ids()); - std::vector topo_sorted_nodes; + // Maintain a mapping from node id to node outputs. + using NodeOutputs = std::vector; + std::vector output_registry(graph_->num_node_ids()); + auto output_registry_cleanup = gtl::MakeCleanup([&output_registry] { + for (const NodeOutputs& outputs : output_registry) { + for (const TensorValue& value : outputs) { + CHECK(!value.is_ref()); + delete value.tensor; + } + } + }); + // XLA requires determinism, generate a stable ordering from DFS. + std::vector topo_sorted_nodes; GetReversePostOrder(*graph_, &topo_sorted_nodes, /*stable_comparator=*/NodeComparatorName()); @@ -94,30 +106,22 @@ Status GraphCompiler::Compile() { PartiallySetupParams(¶ms); for (Node* n : topo_sorted_nodes) { - // Set up bindings. - NodeBinding& binding = bindings[n->id()]; - binding.node = n; - Status s = flib_->CreateKernel(n->def(), &binding.op_kernel); - binding.output_attrs.resize(n->num_outputs()); + OpKernel* op_kernel_raw = nullptr; + Status s = flib_->CreateKernel(n->def(), &op_kernel_raw); + // Transfer ownership of the kernel to a local smart pointer. + std::unique_ptr op_kernel(op_kernel_raw); + if (!s.ok()) { - binding.op_kernel = nullptr; s = AttachDef(s, *n); LOG(ERROR) << "Executor failed to create kernel. " << s; return s; } - } - // Bindings are initialized by the size of graph_->num_node_ids. However, the - // graph may contain dead nodes that still hold a valid node id. Thus - // graph_->num_node_ids could be larger than number of topo sorted nodes. - TF_RET_CHECK(bindings.size() >= topo_sorted_nodes.size()); - - for (Node* n : topo_sorted_nodes) { TF_RET_CHECK(!n->IsRecv() && !n->IsSend() && !n->IsSwitch()) << "Not supported node: " << n->DebugString(); - NodeBinding& binding = bindings[n->id()]; - params.op_kernel = binding.op_kernel; - params.output_attr_array = binding.output_attrs.data(); + params.op_kernel = op_kernel.get(); + gtl::InlinedVector output_attr(n->num_outputs()); + params.output_attr_array = output_attr.data(); // tensor_inputs_ is a buffer reused across graph traversal. We clean up and // reinitialize the buffer before we visit a new node. @@ -128,8 +132,10 @@ Status GraphCompiler::Compile() { for (auto* e : n->in_edges()) { if (e->IsControlEdge()) continue; Node* src = e->src(); - tensor_inputs_[e->dst_input()] = - bindings[src->id()].tensor_values[e->src_output()]; + TF_RET_CHECK(src->id() < output_registry.size()); + const NodeOutputs& src_outputs = output_registry[src->id()]; + + tensor_inputs_[e->dst_input()] = src_outputs[e->src_output()]; } OpKernelContext op_context(¶ms, n->num_outputs()); @@ -143,24 +149,15 @@ Status GraphCompiler::Compile() { // Set up outputs. Also check if outputs from the previous computation is // valid. + NodeOutputs& outputs = output_registry[n->id()]; + outputs.resize(n->num_outputs()); for (int o = 0; o < n->num_outputs(); ++o) { - const auto tensor_val = op_context.release_output(o); - if (*op_context.is_output_dead() || tensor_val.tensor == nullptr) { + outputs[o] = op_context.release_output(o); + if (*op_context.is_output_dead() || outputs[o].tensor == nullptr) { return errors::Internal("Missing xla_context ", o, "-th output from ", (*op_context.is_output_dead() ? "(dead)" : ""), SummarizeNode(*n)); } - binding.tensor_values.push_back(tensor_val); - } - } - - // Clean up tensor data and op kernels. - for (NodeBinding& binding : bindings) { - delete binding.op_kernel; - for (auto& t : binding.tensor_values) { - if (!t.is_ref()) { - delete t.tensor; - } } } return Status::OK(); diff --git a/tensorflow/compiler/tf2xla/graph_compiler.h b/tensorflow/compiler/tf2xla/graph_compiler.h index ccf9351642fb21ab8f14bedd616fdb92215a6492..ba00160b6d78c1e55cc2e053cd5285344e0179fb 100644 --- a/tensorflow/compiler/tf2xla/graph_compiler.h +++ b/tensorflow/compiler/tf2xla/graph_compiler.h @@ -69,20 +69,6 @@ class GraphCompiler { Status Compile(); private: - // NodeBinding is a wrapper on a `Node` that also contains computed - // TensorValue. - struct NodeBinding { - const Node* node; - // Kernel for this node, to be filled by CreateKernel. - // TODO(yunxing): Switching this to unique_ptr and understand why it crashes - // on GPU devices. - OpKernel* op_kernel; - // Output values of this node. - std::vector tensor_values; - // Attributes of the outputs. - gtl::InlinedVector output_attrs; - }; - // Partially sets params. This partially set params can be reused // across multple nodes visit. void PartiallySetupParams(OpKernelContext::Params* params); diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD index f44d61de686278a46b6780eaa974a7939d42a481..2b43e313eb42c288b891f97c0b6cd3cacdc77711 100644 --- a/tensorflow/compiler/tf2xla/kernels/BUILD +++ b/tensorflow/compiler/tf2xla/kernels/BUILD @@ -5,7 +5,6 @@ package( ) load("//tensorflow:tensorflow.bzl", "tf_kernel_library") -load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts") tf_kernel_library( name = "xla_ops", @@ -83,6 +82,7 @@ tf_kernel_library( "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/compiler/tf2xla/ops:sendrecv_ops", "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:client_library", @@ -152,6 +152,7 @@ cc_library( srcs = ["index_ops_kernel_argmax_float_1d.cc"], visibility = ["//visibility:public"], deps = [ + "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry", "//tensorflow/core:framework_lite", "//third_party/eigen3", ], @@ -163,6 +164,7 @@ cc_library( srcs = ["index_ops_kernel_argmax_float_2d.cc"], visibility = ["//visibility:public"], deps = [ + "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry", "//tensorflow/core:framework_lite", "//third_party/eigen3", ], diff --git a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc index 16b778bca439b9236498945f132e8095baeb71c1..73ccc151c1d6bdf70105badd962903297f090abe 100644 --- a/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/batch_matmul_op.cc @@ -77,7 +77,13 @@ class BatchMatMulOp : public XlaOpKernel { xla::ComputationBuilder* builder = ctx->builder(); xla::ComputationDataHandle x_handle = ctx->Input(0); + if (BaseType(input_type(0)) == DT_COMPLEX64 && adj_x_) { + x_handle = builder->Conj(x_handle); + } xla::ComputationDataHandle y_handle = ctx->Input(1); + if (BaseType(input_type(1)) == DT_COMPLEX64 && adj_y_) { + y_handle = builder->Conj(y_handle); + } // Reshape input tensors into 3D tensors by flattening the batch // dimensions. This makes it easier to unroll the batch dimension. diff --git a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc index d635507989bbf78a073be8a50d943dba8688438e..1de91924326464338352b1ac9edf77141f25ad35 100644 --- a/tensorflow/compiler/tf2xla/kernels/binary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/binary_ops.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Native XLA implementations of simple unary Ops +// Native XLA implementations of simple binary Ops #include "tensorflow/compiler/tf2xla/kernels/cwise_ops.h" #include "tensorflow/compiler/tf2xla/xla_helpers.h" @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/client_library.h" #include "tensorflow/compiler/xla/client/computation_builder.h" #include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" namespace tensorflow { namespace { @@ -50,6 +51,9 @@ XLA_MAKE_BINARY(Sub, b->Sub(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Mul, b->Mul(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Div, b->Div(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(Atan2, b->Atan2(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(Complex, b->Complex(lhs, rhs, extend_dimensions)); + // Implementation of FloorDiv. Pseudo-code: // if ((x < 0) != (y < 0)) { // T abs_x = std::abs(x); @@ -98,6 +102,13 @@ XLA_MAKE_BINARY(FloorMod, XLA_MAKE_BINARY(BitwiseAnd, b->And(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(BitwiseOr, b->Or(lhs, rhs, extend_dimensions)); + +XLA_MAKE_BINARY(LeftShift, b->ShiftLeft(lhs, rhs, extend_dimensions)); +XLA_MAKE_BINARY(RightShift, + (DataTypeIsUnsigned(ctx->input_type(0)) + ? b->ShiftRightLogical(lhs, rhs, extend_dimensions) + : b->ShiftRightArithmetic(lhs, rhs, extend_dimensions))); + XLA_MAKE_BINARY(LogicalAnd, b->And(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(LogicalOr, b->Or(lhs, rhs, extend_dimensions)); XLA_MAKE_BINARY(Mod, b->Rem(lhs, rhs, extend_dimensions)); @@ -164,8 +175,12 @@ class ApproximateEqualOp : public XlaOpKernel { // Computes the max of the scalar input x and 0. void Compile(XlaOpKernelContext* ctx) override { xla::ComputationBuilder* b = ctx->builder(); - auto result = b->Lt(b->Abs(b->Sub(ctx->Input(0), ctx->Input(1))), - XlaHelpers::FloatLiteral(b, input_type(0), tolerance_)); + auto abs = b->Abs(b->Sub(ctx->Input(0), ctx->Input(1))); + auto abs_shape = b->GetShape(abs); + OP_REQUIRES_OK(ctx, abs_shape.status()); + auto abs_type = abs_shape.ValueOrDie()->element_type(); + auto result = b->Lt( + abs, b->ConvertElementType(b->ConstantR0(tolerance_), abs_type)); ctx->SetOutput(0, result); } diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc index 2331520230176fce7646d89140851fe37aee5fda..43a6a747c6bcc441f33f276fde4a66f367d99731 100644 --- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/primitive_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" namespace tensorflow { @@ -40,6 +41,11 @@ class CastOp : public XlaOpKernel { output = input; } else if (dst_dtype_ == DT_BOOL) { output = builder->Ne(input, XlaHelpers::Zero(builder, src_dtype_)); + } else if (xla::primitive_util::IsComplexType(src_type_) && + !xla::primitive_util::IsComplexType(dst_type_)) { + // As in cast_op.h, we replicate the numpy behavior of truncating the + // imaginary part. + output = builder->ConvertElementType(builder->Real(input), dst_type_); } else { output = builder->ConvertElementType(input, dst_type_); } diff --git a/tensorflow/compiler/tf2xla/kernels/gather_op.cc b/tensorflow/compiler/tf2xla/kernels/gather_op.cc index db449ec3451d90fe8dce2bef5bea3795dd908277..e420f21ca33fe7de9b33f404ce04eae62d9c041e 100644 --- a/tensorflow/compiler/tf2xla/kernels/gather_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/gather_op.cc @@ -192,7 +192,7 @@ void GatherOpDynamicSlice::Compile(XlaOpKernelContext* context) { errors::InvalidArgument("indices must be int32 or int64")); xla::ComputationDataHandle gather = XlaComputeGatherDynamicSlice( - context, input, input_shape, indices, indices_shape, axis, DT_FLOAT, + context, input, input_shape, indices, indices_shape, axis, input_type(0), index_type, builder); context->SetOutput(0, gather); } diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc index db7d556630a04d93a7eee308117dd429b8af26d1..b8769b3ea2be0a791d9c3e5e7acd8b6184442af2 100644 --- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/compiler/tf2xla/xla_op_kernel.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/arithmetic.h" +#include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -82,16 +83,24 @@ void XlaArgMinMaxOp::Compile(XlaOpKernelContext* ctx) { std::iota(broadcast_dims.begin(), broadcast_dims.begin() + axis, 0); std::iota(broadcast_dims.begin() + axis, broadcast_dims.end(), axis + 1); // Compute a mask that has 1s for elements equal to the maximum. - xla::ComputationDataHandle mask = b->ConvertElementType( + xla::ComputationDataHandle partial_mask = b->ConvertElementType( b->Eq(input, input_max, broadcast_dims), xla_index_type); - // Multiply by the vector [0, 1, 2, ...] to convert each 1 into its index. - // TODO(phawkins): add a bitwise And operator to HLO, use a bitwise and - // instead of a multiplication here. + // In order to make identity elements for a bitwise And, we: + // Left shift the 1 to the leftmost bit, yielding 0x10...0 + // Arithmetic right shift the 1 back to the rightmost bit, yielding 0xFF...F + int32 bits_in_type = + xla::ShapeUtil::ByteSizeOfPrimitiveType(xla_index_type) * 8 - 1; + xla::ComputationDataHandle shift_amount = + XlaHelpers::IntegerLiteral(b, index_type, bits_in_type); + xla::ComputationDataHandle full_mask = b->ShiftRightArithmetic( + b->ShiftLeft(partial_mask, shift_amount), shift_amount); + + // And with the vector [0, 1, 2, ...] to convert each 0xFF...F into its index. xla::ComputationDataHandle iota; OP_REQUIRES_OK(ctx, XlaHelpers::Iota(b, index_type, axis_size, &iota)); xla::ComputationDataHandle product = - b->Mul(mask, iota, /*broadcast_dimensions=*/{axis}); + b->And(full_mask, iota, /*broadcast_dimensions=*/{axis}); // If there are multiple maximum elements, choose the one with the highest // index. diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc index afbd64ca5038378d48744d6d773e0dfb1376e1f9..47cf8c6675bc120653c2a5ab6d4b07376dc382ee 100644 --- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc +++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_1d.cc @@ -16,6 +16,7 @@ limitations under the License. #define EIGEN_USE_THREADS #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/dynamic_annotations.h" #include "tensorflow/core/platform/macros.h" @@ -47,3 +48,5 @@ EIGEN_STRONG_INLINE void argmax_float_1d_xla_impl(void* out, void** data) { extern "C" void TF_EXPORT argmax_float_1d_xla_impl(void* out, void** data) { tensorflow::argmax_float_1d_xla_impl(out, data); } + +REGISTER_CUSTOM_CALL_TARGET(argmax_float_1d_xla_impl); diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc index 841ff2f4df79fdd790ee3aace9e38aaeb01a3080..9b83392d8fbe461970603fbadee76e8d71b1ebd0 100644 --- a/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc +++ b/tensorflow/compiler/tf2xla/kernels/index_ops_kernel_argmax_float_2d.cc @@ -16,6 +16,7 @@ limitations under the License. #define EIGEN_USE_THREADS #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/dynamic_annotations.h" #include "tensorflow/core/platform/macros.h" @@ -49,3 +50,5 @@ EIGEN_STRONG_INLINE void argmax_float_2d_xla_impl(void* out, void** data) { extern "C" void TF_EXPORT argmax_float_2d_xla_impl(void* out, void** data) { tensorflow::argmax_float_2d_xla_impl(out, data); } + +REGISTER_CUSTOM_CALL_TARGET(argmax_float_2d_xla_impl); diff --git a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc index 5c799a0e4f86db04dc966411e0c917387186ce59..fcef497e5845d9080bc83b54e92dcf2fdecf5f12 100644 --- a/tensorflow/compiler/tf2xla/kernels/matmul_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/matmul_op.cc @@ -23,6 +23,9 @@ limitations under the License. namespace tensorflow { namespace { +constexpr std::array kMatmulTypes = { + {DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64}}; + class MatMulOp : public XlaOpKernel { public: explicit MatMulOp(OpKernelConstruction* ctx, bool is_sparse = false) @@ -73,7 +76,7 @@ class MatMulOp : public XlaOpKernel { bool transpose_b_; }; -REGISTER_XLA_OP(Name("MatMul").TypeConstraint("T", kFloatTypes), MatMulOp); +REGISTER_XLA_OP(Name("MatMul").TypeConstraint("T", kMatmulTypes), MatMulOp); class SparseMatMulOp : public MatMulOp { public: diff --git a/tensorflow/compiler/tf2xla/kernels/training_ops.cc b/tensorflow/compiler/tf2xla/kernels/training_ops.cc index 82ae0df5cc501cf1b51c2b25b9330d582fbdc44c..5534d1bfa1338c7fe3647cd6aa281c4907dfdf8c 100644 --- a/tensorflow/compiler/tf2xla/kernels/training_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/training_ops.cc @@ -37,8 +37,9 @@ class ResourceApplyGradientDescent : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->AssignVariable(0, ctx->input_type(1), handle)); } }; -REGISTER_XLA_OP(Name("ResourceApplyGradientDescent"), - ResourceApplyGradientDescent); +REGISTER_XLA_OP( + Name("ResourceApplyGradientDescent").TypeConstraint("T", kFloatTypes), + ResourceApplyGradientDescent); class ResourceApplyMomentum : public XlaOpKernel { public: @@ -109,7 +110,8 @@ class ResourceApplyMomentum : public XlaOpKernel { private: bool use_nesterov_; }; -REGISTER_XLA_OP(Name("ResourceApplyMomentum"), ResourceApplyMomentum); +REGISTER_XLA_OP(Name("ResourceApplyMomentum").TypeConstraint("T", kFloatTypes), + ResourceApplyMomentum); class ResourceApplyAdagrad : public XlaOpKernel { public: @@ -163,7 +165,8 @@ class ResourceApplyAdagrad : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->AssignVariable(1, type, accum)); } }; -REGISTER_XLA_OP(Name("ResourceApplyAdagrad"), ResourceApplyAdagrad); +REGISTER_XLA_OP(Name("ResourceApplyAdagrad").TypeConstraint("T", kFloatTypes), + ResourceApplyAdagrad); class ResourceApplyAdam : public XlaOpKernel { public: @@ -263,7 +266,8 @@ class ResourceApplyAdam : public XlaOpKernel { private: DataType dtype_; }; -REGISTER_XLA_OP(Name("ResourceApplyAdam"), ResourceApplyAdam); +REGISTER_XLA_OP(Name("ResourceApplyAdam").TypeConstraint("T", kFloatTypes), + ResourceApplyAdam); class ResourceApplyRMSProp : public XlaOpKernel { public: @@ -362,7 +366,8 @@ class ResourceApplyRMSProp : public XlaOpKernel { OP_REQUIRES_OK(ctx, ctx->AssignVariable(2, type, new_mom)); } }; -REGISTER_XLA_OP(Name("ResourceApplyRMSProp"), ResourceApplyRMSProp); +REGISTER_XLA_OP(Name("ResourceApplyRMSProp").TypeConstraint("T", kFloatTypes), + ResourceApplyRMSProp); void CompileFtrl(XlaOpKernelContext* ctx, DataType dtype, bool has_l2_shrinkage) { @@ -500,7 +505,8 @@ class ResourceApplyFtrl : public XlaOpKernel { private: DataType dtype_; }; -REGISTER_XLA_OP(Name("ResourceApplyFtrl"), ResourceApplyFtrl); +REGISTER_XLA_OP(Name("ResourceApplyFtrl").TypeConstraint("T", kFloatTypes), + ResourceApplyFtrl); class ResourceApplyFtrlV2 : public XlaOpKernel { public: @@ -515,7 +521,8 @@ class ResourceApplyFtrlV2 : public XlaOpKernel { private: DataType dtype_; }; -REGISTER_XLA_OP(Name("ResourceApplyFtrlV2"), ResourceApplyFtrlV2); +REGISTER_XLA_OP(Name("ResourceApplyFtrlV2").TypeConstraint("T", kFloatTypes), + ResourceApplyFtrlV2); } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc index 651bbe2b405df66cb6aff1ba7fe3957eba94d610..a266e9013c41b88788dbc99849f01c09f3d61348 100644 --- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc @@ -41,6 +41,12 @@ namespace { }; \ REGISTER_XLA_OP(Name(#NAME), NAME##Op); +XLAJIT_MAKE_UNARY(ComplexAbs, b->Abs(x)); + +XLAJIT_MAKE_UNARY(Angle, b->Atan2(b->Imag(x), b->Real(x))); + +XLAJIT_MAKE_UNARY(Conj, b->Conj(x)); + // Return x if x>0, otherwise -x. XLAJIT_MAKE_UNARY(Abs, b->Abs(x)); @@ -162,6 +168,9 @@ XLAJIT_MAKE_UNARY(Square, b->Mul(x, x)); XLAJIT_MAKE_UNARY(Tan, b->Div(b->Sin(x), b->Cos(x))); XLAJIT_MAKE_UNARY(Tanh, b->Tanh(x)); +XLAJIT_MAKE_UNARY(Real, b->Real(x)); +XLAJIT_MAKE_UNARY(Imag, b->Imag(x)); + #undef XLAJIT_MAKE_UNARY } // namespace diff --git a/tensorflow/compiler/tf2xla/type_util.cc b/tensorflow/compiler/tf2xla/type_util.cc index c6984887766e7778d2f8f2fdbd0d626cf9451d86..1efbe0ffb17dad5332aa700b2e255d4a99fbef72 100644 --- a/tensorflow/compiler/tf2xla/type_util.cc +++ b/tensorflow/compiler/tf2xla/type_util.cc @@ -58,6 +58,9 @@ Status DataTypeToPrimitiveType(DataType data_type, xla::PrimitiveType* type) { case tensorflow::DT_DOUBLE: *type = xla::F64; return Status::OK(); + case tensorflow::DT_COMPLEX64: + *type = xla::C64; + return Status::OK(); case tensorflow::DT_QUINT8: *type = xla::U8; return Status::OK(); diff --git a/tensorflow/compiler/tf2xla/xla_compilation_device.cc b/tensorflow/compiler/tf2xla/xla_compilation_device.cc index 890a9ccb830c75afcb81d28685cc26e4a7ef35f9..fc866a4c0a34712dc3906fb60c13a30909ecffd2 100644 --- a/tensorflow/compiler/tf2xla/xla_compilation_device.cc +++ b/tensorflow/compiler/tf2xla/xla_compilation_device.cc @@ -103,20 +103,17 @@ void XlaCompilationDevice::Compute(OpKernel* op_kernel, DeviceNameUtils::ParseFullName(op_kernel->requested_device(), &parsed), errors::Internal("Unable to parse device name: ", op_kernel->requested_device())); - xla::OpDeviceAssignment assignment; // If no device ID assignment is found, XLA is free to use whatever device it // wants. In practice this usually has the effect of placing things on // device 0. if (parsed.has_id) { - assignment.set_has_device(true); - assignment.set_device(parsed.id); + b->SetSharding(xla::ShardingBuilder::AssignDevice(parsed.id)); } - b->SetDeviceAssignment(assignment); op_kernel->Compute(context); b->ClearOpMetadata(); - b->ClearDeviceAssignment(); + b->ClearSharding(); VLOG(4) << "Done"; } diff --git a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h index 01e6b4c071a057429b78171b1c6ff2f38bb85590..f49a7889222ff989144217ab10b27595f89e4311 100644 --- a/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h +++ b/tensorflow/compiler/tf2xla/xla_compiled_cpu_function.h @@ -56,7 +56,7 @@ class XlaCompiledCpuFunction { const void** args, void** temps)>; // StaticData represents the state necessary to run an XLA-compiled - // function. For JIT this is backed by data in XlaCompiledCpuFunctionJit; for + // function. For JIT this is backed by data in XlaJitCompiledCpuFunction; for // AOT this is backed by data compiled into the object file. struct StaticData { // The raw function to call. diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc index a82ef02e32c78373ec2aa56558f525d7b825d861..e49663b8b047fb5f2c9ba17fa0aa032a673e7ed7 100644 --- a/tensorflow/compiler/tf2xla/xla_compiler.cc +++ b/tensorflow/compiler/tf2xla/xla_compiler.cc @@ -177,7 +177,9 @@ Status XlaCompiler::CompileFunction( const FunctionBody* fbody; TF_RETURN_IF_ERROR(FindFunctionBody(function, &fbody)); - TF_RETURN_IF_ERROR(CheckSignature(fbody->arg_types, args)); + TF_RETURN_WITH_CONTEXT_IF_ERROR( + CheckSignature(fbody->arg_types, args), + "Signature check failure while compiling: ", function.name()); std::unique_ptr graph(new Graph(options_.flib_def)); CopyGraph(*fbody->graph, graph.get()); diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc index f59b83cfdd778209935970981a1463d350a64be6..de5ad5f176536e1453da518b96ee755c7f1e8fdc 100644 --- a/tensorflow/compiler/tf2xla/xla_helpers.cc +++ b/tensorflow/compiler/tf2xla/xla_helpers.cc @@ -97,6 +97,9 @@ xla::ComputationDataHandle XlaHelpers::IntegerLiteral( case xla::F64: literal = *xla::Literal::CreateR0(value); break; + case xla::C64: + literal = *xla::Literal::CreateR0(value); + break; case xla::PRED: LOG(FATAL) << "pred element type is not integral"; case xla::S16: @@ -132,6 +135,9 @@ xla::ComputationDataHandle XlaHelpers::FloatLiteral(xla::ComputationBuilder* b, case xla::F64: return b->ConstantR0(value); break; + case xla::C64: + return b->ConstantR0(value); + break; default: LOG(FATAL) << "unhandled element type " << type; } diff --git a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc index 5bee68eefc8d9452b63113c080fc86d39550e899..6d49298a6f3e8a726695fafc42f3c5341fe98b5f 100644 --- a/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc +++ b/tensorflow/compiler/tf2xla/xla_jit_compiled_cpu_function_test.cc @@ -129,5 +129,19 @@ TEST(XlaJitCompiledCpuFunction, Sum) { EXPECT_TRUE(ShapeUtil::Compatible(result0, s32)); } +// Test when a graph compilation terminates early, resources are properly +// reclaimed. +TEST(XlaJitCompiledCpuFunction, SumWithJunkAttr) { + GraphDef graph_def = SumGraph(); + + (*graph_def.mutable_node(2)->mutable_attr())["junk"] = + TypeAttrValue(DT_INT32); + + tf2xla::Config config = SumConfig(); + EXPECT_FALSE(XlaJitCompiledCpuFunction::Compile(graph_def, config, + xla::ExecutableBuildOptions()) + .ok()); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h index 21448686463bddd719340715bcf80987ef332caf..6aee8c91cc01b4382ef867fa8e438eede008ac73 100644 --- a/tensorflow/compiler/tf2xla/xla_op_registry.h +++ b/tensorflow/compiler/tf2xla/xla_op_registry.h @@ -47,14 +47,17 @@ extern const char* const DEVICE_XLA_GPU; constexpr std::array kFloatTypes = { {DT_HALF, DT_FLOAT, DT_DOUBLE}}; -constexpr std::array kNumericTypes = { - {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE}}; +constexpr std::array kNumericTypes = { + {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, + DT_COMPLEX64}}; -constexpr std::array kCpuAllTypes = { - {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kCpuAllTypes = { + {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, + DT_COMPLEX64, DT_BOOL}}; -constexpr std::array kGpuAllTypes = { - {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_BOOL}}; +constexpr std::array kGpuAllTypes = { + {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, + DT_COMPLEX64, DT_BOOL}}; // Class that manages registrations of operators and devices for the XLA JIT. // Not thread-safe. diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 6c4c970ce838400794e9fd4f3bddb829d8a14e5b..660f419e464936b01a3644e69c2f056f998140f5 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -7,7 +7,6 @@ package_group( packages = [ "//tensorflow/compiler/...", "//tensorflow/contrib/tpu/...", - "//tensorflow/contrib/xla_tf_graph/...", ], ) @@ -171,6 +170,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":status", + ":status_macros", ":types", ":xla_data_proto", "//tensorflow/core:lib", @@ -335,12 +335,32 @@ cc_library( ], ) +cc_library( + name = "array", + hdrs = ["array.h"], + deps = [ + ":types", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "array_test", + srcs = ["array_test.cc"], + deps = [ + ":array", + ":test", + "//tensorflow/core:test_main", + ], +) + cc_library( name = "array2d", srcs = ["array2d.cc"], hdrs = ["array2d.h"], visibility = ["//visibility:public"], deps = [ + ":array", ":types", ":util", "//tensorflow/core:lib", @@ -362,6 +382,7 @@ cc_library( hdrs = ["array3d.h"], visibility = [":friends"], deps = [ + ":array", ":types", "//tensorflow/core:lib", ], @@ -383,6 +404,7 @@ cc_library( hdrs = ["array4d.h"], visibility = [":friends"], deps = [ + ":array", ":array2d", ":types", "//tensorflow/core:lib", diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h new file mode 100644 index 0000000000000000000000000000000000000000..ba898d1f4e9100df59c6e4b28824895c5ae6c08a --- /dev/null +++ b/tensorflow/compiler/xla/array.h @@ -0,0 +1,342 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_ARRAY_H_ +#define TENSORFLOW_COMPILER_XLA_ARRAY_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/lib/core/bits.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +// General N dimensional array class with arbitrary value type. +template +class Array { + public: + // Creates a new array with the specified dimensions. + explicit Array(tensorflow::gtl::ArraySlice sizes) + : Array(sizes, T()) {} + + // Creates a new array with the specified dimensions and specified value for + // every cell. + Array(tensorflow::gtl::ArraySlice sizes, T value) + : sizes_(sizes.begin(), sizes.end()), values_(new T[num_elements()]) { + Fill(value); + } + + // Creates a 2D array from the given nested initializer list. The outer + // initializer list is the first dimension, the inner is the second dimension. + // For example, {{1, 2, 3}, {4, 5, 6}} results in an array with n1=2 and n2=3. + Array(std::initializer_list> values) + : Array(ToInt64Vector({values.size(), values.begin()->size()})) { + int64 idx = 0; + for (const auto& it1 : values) { + for (const auto& it2 : it1) { + values_[idx] = it2; + ++idx; + } + } + CHECK(idx == num_elements()); + } + + // Creates a 3D array from the given nested initializer list. The outer + // initializer list is the first dimension, and so on. + Array(std::initializer_list>> + values) + : Array(ToInt64Vector({values.size(), values.begin()->size(), + values.begin()->begin()->size()})) { + int64 idx = 0; + for (const auto& it1 : values) { + for (const auto& it2 : it1) { + for (const auto& it3 : it2) { + values_[idx] = it3; + ++idx; + } + } + } + CHECK(idx == num_elements()); + } + + // Creates a 4D array from the given nested initializer list. The outer + // initializer list is the first dimension, and so on. + Array(std::initializer_list< + std::initializer_list>>> + values) + : Array(ToInt64Vector({values.size(), values.begin()->size(), + values.begin()->begin()->size(), + values.begin()->begin()->begin()->size()})) { + int64 idx = 0; + for (const auto& it1 : values) { + for (const auto& it2 : it1) { + for (const auto& it3 : it2) { + for (const auto& it4 : it3) { + values_[idx] = it4; + ++idx; + } + } + } + } + CHECK(idx == num_elements()); + } + + Array(const Array& other) + : sizes_(other.sizes_), values_(new T[num_elements()]) { + std::copy(&other.values_[0], &other.values_[0] + num_elements(), + &values_[0]); + } + + Array& operator=(const Array& other) { + sizes_ = other.sizes_; + values_.reset(new T[num_elements()]); + std::copy(&other.values_[0], &other.values_[0] + num_elements(), + &values_[0]); + return *this; + } + + // Fills the array with the specified value. + void Fill(const T& value) { + std::fill(&values_[0], &values_[0] + num_elements(), value); + } + + // Fills the array with sequentially increasing values. + void FillIota(const T& value) { + std::iota(&values_[0], &values_[0] + num_elements(), value); + } + + // Fills the array with the sequence i*multiplier for i=0,1,... + void FillWithMultiples(const T& multiplier) { + for (int64 i = 0; i < num_elements(); ++i) { + values_[i] = i * multiplier; + } + } + + // Fills the array with random normal variables with the specified mean. + void FillRandom(const T& value, const double mean = 0.0, + const int seed = 12345) { + std::mt19937 g(seed); + std::normal_distribution distribution(mean, + static_cast(value)); + for (int64 i = 0; i < num_elements(); ++i) { + values_[i] = static_cast(distribution(g)); + } + } + + // Sets all the values in the array to values specified in the container. + template > + void SetValues(const Container& container) { + CHECK_EQ(std::distance(std::begin(container), std::end(container)), + num_elements()); + std::copy(std::begin(container), std::end(container), &values_[0]); + } + + // Invokes a callback with the (indices, value_ptr) for each cell in the + // array. + void Each(std::function, T*)> f) { + std::vector index(sizes_.size()); + for (int64 i = 0; i < num_elements(); ++i, next_index(&index)) { + f(index, &values_[i]); + } + } + + // Invokes a callback with the (indices, value) for each cell in the array. + void Each( + std::function, T)> f) const { + std::vector index(sizes_.size()); + for (int64 i = 0; i < num_elements(); ++i, next_index(&index)) { + f(index, values_[i]); + } + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + template + const T& operator()(Dims... dims) const { + // We are using a std::array to avoid having to allocate memory in this + // function for performance reasons. + std::array indexes{{static_cast(dims)...}}; + return values_[calculate_index(indexes)]; + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + template + T& operator()(Dims... dims) { + // We are using a std::array to avoid having to allocate memory in this + // function for performance reasons. + std::array indexes{{static_cast(dims)...}}; + return values_[calculate_index(indexes)]; + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + const T& operator()(tensorflow::gtl::ArraySlice indexes) const { + return values_[calculate_index(indexes)]; + } + + // Returns the value at the cell specified by the indexes. The number of + // arguments have to match with the number of dimensions for the array. + T& operator()(tensorflow::gtl::ArraySlice indexes) { + return values_[calculate_index(indexes)]; + } + + // Low-level accessor for stuff like memcmp, handle with care. Returns pointer + // to the underlying storage of the array (similarly to std::vector::data()). + T* data() const { + // TODO(tberghammer): Get rid of the const_cast. Currently it is needed + // because the Eigen backend needs a non-const pointers even for reading + // from the array. + return const_cast(this)->values_.get(); + } + + // Returns the size of the dimension at the given index. + int64 dim(int64 n) const { + CHECK(n < sizes_.size()); + return sizes_[n]; + } + + // Returns a vector containing the dimensions of the array. + const std::vector& dimensions() const { return sizes_; } + + int64 num_dimensions() const { return sizes_.size(); } + + // Returns the total number of elements in the array. + int64 num_elements() const { + return std::accumulate(sizes_.begin(), sizes_.end(), 1, + std::multiplies()); + } + + const T* begin() const { return &values_[0]; } + T* begin() { return &values_[0]; } + const T* end() const { return &values_[num_elements()]; } + T* end() { return &values_[num_elements()]; } + + bool operator==(const Array& other) const { + if (sizes_.size() != other.sizes_.size()) { + return false; + } + for (int64 i = 0; i < sizes_.size(); ++i) { + if (sizes_[i] != other.sizes_[i]) { + return false; + } + } + for (int64 i = 0; i < num_elements(); ++i) { + if (values_[i] != other.values_[i]) { + return false; + } + } + return true; + } + + bool operator!=(const Array& other) const { return !(*this == other); } + + // Returns a string representation of the array suitable for debugging. + string ToString() const { + std::vector pieces; + std::vector index(sizes_.size()); + do { + // Emit leading spaces and opening square brackets + if (index.back() == 0) { + for (int64 i = sizes_.size() - 1; i >= 0; --i) { + if (i == 0 || index[i - 1] != 0) { + for (int64 j = 0; j < sizes_.size(); ++j) { + pieces.push_back(j < i ? " " : "["); + } + break; + } + } + } + + pieces.push_back( + tensorflow::strings::AlphaNum(values_[calculate_index(index)]) + .data()); + + // Emit comma if it isn't the last element + if (index.back() != sizes_.back() - 1) { + pieces.push_back(", "); + } + + // Emit closing square brackets + for (int64 i = sizes_.size() - 1; i >= 0; --i) { + if (index[i] != sizes_[i] - 1) { + break; + } + pieces.push_back("]"); + if (i != 0 && index[i - 1] != sizes_[i - 1] - 1) { + pieces.push_back(",\n"); + } + } + } while (next_index(&index)); + return tensorflow::str_util::Join(pieces, ""); + } + + private: + // Converts an initializer_list of type U to a vector of type int64. Used by + // the initializer list based constructors to convert the size type into int64 + // to be passed to the size based constructor. + template + static std::vector ToInt64Vector( + const std::initializer_list& data) { + return std::vector(data.begin(), data.end()); + } + + // Returns the linear index from the list of per-dimension indexes. Function + // is templated so can be used with an std::array from operator() to avoid + // memory allocation. + template + int64 calculate_index(const U& indexes) const { + CHECK_EQ(sizes_.size(), indexes.size()); + int64 index = 0; + for (int64 i = 0; i < sizes_.size(); ++i) { + index *= sizes_[i]; + index += indexes[i]; + } + return index; + } + + // Advances the specified set of indexes and returns true if we haven't + // wrapped around (i.e. result isn't {0, 0, ...}). + bool next_index(std::vector* index) const { + CHECK_EQ(index->size(), sizes_.size()); + for (int64 i = sizes_.size() - 1; i >= 0; --i) { + (*index)[i]++; + if ((*index)[i] < sizes_[i]) { + return true; + } + (*index)[i] = 0; + } + return false; + } + + std::vector sizes_; + std::unique_ptr values_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_ARRAY_H_ diff --git a/tensorflow/compiler/xla/array2d.h b/tensorflow/compiler/xla/array2d.h index 2737764cbda87298599d7005c237a2093cbaba4a..bb85fbee9b97fd6b9b0bf7223a9b820989dcbfa7 100644 --- a/tensorflow/compiler/xla/array2d.h +++ b/tensorflow/compiler/xla/array2d.h @@ -24,6 +24,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/lib/core/bits.h" #include "tensorflow/core/lib/strings/str_util.h" @@ -34,93 +35,30 @@ limitations under the License. namespace xla { -// Simple 2D array structure. -// -// The data layout in major-to-minor order is: n1, n2. template -class Array2D { +class Array2D : public Array { public: - // Creates an empty array. - Array2D() : n1_(0), n2_(0) {} + Array2D() : Array(std::vector{0, 0}) {} - // Creates an array of dimensions n1 x n2, uninitialized values. Array2D(const int64 n1, const int64 n2) - : n1_(n1), n2_(n2), values_(new T[n1 * n2]()) { - Fill(T()); - } + : Array(std::vector{n1, n2}) {} - // Creates an array of dimensions n1 x n2, initialized to value. Array2D(const int64 n1, const int64 n2, const T value) - : n1_(n1), n2_(n2), values_(new T[n1 * n2]()) { - Fill(value); - } + : Array({n1, n2}, value) {} // Creates an array from the given nested initializer list. The outer // initializer list is the first dimension; the inner is the second dimension. // For example, {{1, 2, 3}, {4, 5, 6}} results in an array with n1=2 and n2=3. Array2D(std::initializer_list> values) - : Array2D(values.size(), values.begin()->size()) { - int64 n1 = 0; - for (auto n1_it = values.begin(); n1_it != values.end(); ++n1_it, ++n1) { - int64 n2 = 0; - for (auto n2_it = n1_it->begin(); n2_it != n1_it->end(); ++n2_it, ++n2) { - (*this)(n1, n2) = *n2_it; - } - } - } + : Array(values) {} - Array2D(const Array2D& other) : Array2D(other.n1(), other.n2()) { - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - } - - Array2D& operator=(const Array2D& other) { - n1_ = other.n1(); - n2_ = other.n2(); - values_.reset(new T[num_elements()]); - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - return *this; - } + Array2D(const Array2D& other) : Array(other) {} - T& operator()(const int64 i1, const int64 i2) { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - return values_[i1 * n2_ + i2]; - } + int64 n1() const { return this->dim(0); } + int64 n2() const { return this->dim(1); } - const T& operator()(const int64 i1, const int64 i2) const { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - return values_[i1 * n2_ + i2]; - } - - // Access to the array's dimensions. height() and width() provide the - // canonical interpretation of the array n1 x n2 having n1 rows of n2 columns - // each (height is number of rows; width is number of columns). - int64 n1() const { return n1_; } - int64 n2() const { return n2_; } - int64 height() const { return n1_; } - int64 width() const { return n2_; } - int64 num_elements() const { return n1_ * n2_; } - - // Low-level accessor for stuff like memcmp, handle with care. Returns pointer - // to the underlying storage of the array (similarly to std::vector::data()). - T* data() const { return const_cast(this)->values_.get(); } - - // Fills the array with the given value. - void Fill(const T& value) { - std::fill(&values_[0], &values_[0] + num_elements(), value); - } - - // Applies f to all cells in this array, in row-major order. - void Each(std::function f) { - for (int64 i0 = 0; i0 < n1(); ++i0) { - for (int64 i1 = 0; i1 < n2(); ++i1) { - f(i0, i1, &(*this)(i0, i1)); - } - } - } + int64 height() const { return this->dim(0); } + int64 width() const { return this->dim(1); } // Fills the array with a pattern of values of the form: // @@ -136,55 +74,14 @@ class Array2D { } } - // Fills the array with random normal variables of deviation value. - void FillRandom(const T& value, const double mean = 0.0, - const int seed = 12345) { - std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(value)); - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); - } - } - - // Returns a readable string representation of the array. - string ToString() const { - std::vector pieces = {"["}; - for (int64 row = 0; row < height(); ++row) { - pieces.push_back("["); - for (int64 col = 0; col < width(); ++col) { - pieces.push_back(tensorflow::strings::StrCat((*this)(row, col))); - pieces.push_back(", "); - } - pieces.pop_back(); - pieces.push_back("]"); - pieces.push_back(",\n "); - } - pieces.pop_back(); - pieces.push_back("]"); - return tensorflow::str_util::Join(pieces, ""); - } - - bool operator==(const Array2D& other) const { - if (n1() != other.n1() || n2() != other.n2()) { - return false; - } + // Applies f to all cells in this array, in row-major order. + void Each(std::function f) { for (int64 i0 = 0; i0 < n1(); ++i0) { for (int64 i1 = 0; i1 < n2(); ++i1) { - if ((*this)(i0, i1) != other(i0, i1)) { - return false; - } + f(i0, i1, &(*this)(i0, i1)); } } - return true; } - - bool operator!=(const Array2D& other) const { return !(*this == other); } - - private: - int64 n1_; - int64 n2_; - std::unique_ptr values_; }; // Returns a linspace-populated Array2D in the range [from, to] (inclusive) diff --git a/tensorflow/compiler/xla/array3d.h b/tensorflow/compiler/xla/array3d.h index 124ccd1975b3a9ab047e9bbbfb38921fe7386fe4..e9449f01ad69a5722f53cce09e2884e20a0def5a 100644 --- a/tensorflow/compiler/xla/array3d.h +++ b/tensorflow/compiler/xla/array3d.h @@ -24,6 +24,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -32,22 +33,16 @@ limitations under the License. namespace xla { // Simple 3D array structure. -// -// The data layout in major-to-minor order is: n1, n2, n3. template -class Array3D { +class Array3D : public Array { public: // Creates an array of dimensions n1 x n2 x n3, uninitialized values. Array3D(const int64 n1, const int64 n2, const int64 n3) - : n1_(n1), n2_(n2), n3_(n3), values_(new T[n1 * n2 * n3]) { - Fill(T()); - } + : Array(std::vector{n1, n2, n3}) {} // Creates an array of dimensions n1 x n2 x n3, initialized to value. Array3D(const int64 n1, const int64 n2, const int64 n3, const T value) - : n1_(n1), n2_(n2), n3_(n3), values_(new T[n1 * n2 * n3]) { - Fill(value); - } + : Array(std::vector{n1, n2, n3}, value) {} // Creates an array from the given nested initializer list. The outer // initializer list is the first dimension, and so on. @@ -58,84 +53,11 @@ class Array3D { // results in an array with n1=3, n2=4, n3=2. Array3D(std::initializer_list>> values) - : Array3D(values.size(), values.begin()->size(), - values.begin()->begin()->size()) { - int64 n1 = 0; - for (auto n1_it = values.begin(); n1_it != values.end(); ++n1_it, ++n1) { - int64 n2 = 0; - for (auto n2_it = n1_it->begin(); n2_it != n1_it->end(); ++n2_it, ++n2) { - int64 n3 = 0; - for (auto n3_it = n2_it->begin(); n3_it != n2_it->end(); - ++n3_it, ++n3) { - (*this)(n1, n2, n3) = *n3_it; - } - } - } - } + : Array(values) {} - Array3D(const Array3D& other) - : Array3D(other.n1(), other.n2(), other.n3()) { - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - } - - Array3D& operator=(const Array3D& other) { - n1_ = other.n1(); - n2_ = other.n2(); - n3_ = other.n3(); - values_.reset(new T[num_elements()]); - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - return *this; - } - - T& operator()(const int64 i1, const int64 i2, const int64 i3) { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - CHECK_LT(i3, n3_); - return values_[i1 * n2_ * n3_ + i2 * n3_ + i3]; - } - - const T& operator()(const int64 i1, const int64 i2, const int64 i3) const { - CHECK_LT(i1, n1_); - CHECK_LT(i2, n2_); - CHECK_LT(i3, n3_); - return values_[i1 * n2_ * n3_ + i2 * n3_ + i3]; - } - - // Access to the array's dimensions. - int64 n1() const { return n1_; } - int64 n2() const { return n2_; } - int64 n3() const { return n3_; } - int64 num_elements() const { return n1_ * n2_ * n3_; } - - // Fills the array with the given value. - void Fill(const T& value) { - std::fill(&values_[0], &values_[0] + num_elements(), value); - } - - // Fills the array with sequentially increasing values. - void FillIota(const T& value) { - std::iota(&values_[0], &values_[0] + num_elements(), value); - } - - // Fills the array with random normal values with a mean of 0 and standard - // deviation of value. - void FillRandom(const T& value, const double mean = 0.0, - const int seed = 12345) { - std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(value)); - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); - } - } - - private: - int64 n1_; - int64 n2_; - int64 n3_; - std::unique_ptr values_; + int64 n1() const { return this->dim(0); } + int64 n2() const { return this->dim(1); } + int64 n3() const { return this->dim(2); } }; } // namespace xla diff --git a/tensorflow/compiler/xla/array4d.h b/tensorflow/compiler/xla/array4d.h index 4c7fce1aaf1faf4bd08bca38bc8eb2b47303b575..f8b2b2afe5fed9c465c2a1f39308b7f44311b16a 100644 --- a/tensorflow/compiler/xla/array4d.h +++ b/tensorflow/compiler/xla/array4d.h @@ -26,6 +26,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/core/lib/gtl/array_slice.h" @@ -53,23 +54,15 @@ namespace xla { // more than one name is given above. See operator() for the exact // calculation of 1d indices from 4d indices. template -class Array4D { +class Array4D : public Array { public: // Creates a 4D array, uninitialized values. Array4D(int64 planes, int64 depth, int64 height, int64 width) - : planes_(planes), - depth_(depth), - height_(height), - width_(width), - values_(new T[planes * depth * height * width]) { - Fill(T()); - } + : Array(std::vector{planes, depth, height, width}) {} // Creates a 4D array, initialized to value. Array4D(int64 planes, int64 depth, int64 height, int64 width, T value) - : Array4D(planes, depth, height, width) { - Fill(value); - } + : Array(std::vector{planes, depth, height, width}, value) {} // Creates a 4D array, filled with values. // @@ -80,144 +73,26 @@ class Array4D { Array4D(int64 planes, int64 depth, int64 height, int64 width, const Container& values) : Array4D(planes, depth, height, width) { - SetValues(values); + this->SetValues(values); } // Construct an Array4D with the given nested initializer list. Array4D(std::initializer_list>>> values) - : Array4D(values.size(), values.begin()->size(), - values.begin()->begin()->size(), - values.begin()->begin()->begin()->size()) { - int64 plane = 0; - for (const auto values_in_plane : values) { - DCHECK_EQ(values_in_plane.size(), depth_); - int64 depth = 0; - for (const auto values_in_depth : values_in_plane) { - DCHECK_EQ(values_in_depth.size(), height_); - int64 height = 0; - for (const auto values_in_height : values_in_depth) { - DCHECK_EQ(values_in_height.size(), width_); - int64 width = 0; - for (const auto element_value : values_in_height) { - (*this)(plane, depth, height, width) = element_value; - ++width; - } - ++height; - } - ++depth; - } - ++plane; - } - } - - Array4D(const Array4D& other) - : Array4D(other.planes(), other.depth(), other.height(), other.width()) { - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - } - - Array4D& operator=(const Array4D& other) { - planes_ = other.planes(); - depth_ = other.depth(); - height_ = other.height(); - width_ = other.width(); - values_.reset(new T[num_elements()]); - std::copy(&other.values_[0], &other.values_[0] + num_elements(), - &values_[0]); - return *this; - } - - T& operator()(int64 plane, int64 depth, int64 height, int64 width) { - CHECK_LT(plane, planes_); - CHECK_LT(depth, depth_); - CHECK_LT(height, height_); - CHECK_LT(width, width_); - return values_[plane * (depth_ * height_ * width_) + - depth * (height_ * width_) + height * (width_) + width]; - } - const T& operator()(int64 plane, int64 depth, int64 height, - int64 width) const { - return const_cast(this)->operator()(plane, depth, height, width); - } - - int64 width() const { return width_; } - int64 height() const { return height_; } - int64 depth() const { return depth_; } - int64 planes() const { return planes_; } + : Array(values) {} // Numerically-named aliases for the various dimensions. This matches the // dimension names used in array3d. - int64 n4() const { return width_; } - int64 n3() const { return height_; } - int64 n2() const { return depth_; } - int64 n1() const { return planes_; } - int64 num_elements() const { return width_ * height_ * depth_ * planes_; } - - // Sets all the values in the array to values. - template > - void SetValues(const Container& container) { - CHECK_EQ(std::distance(std::begin(container), std::end(container)), - num_elements()); - std::copy(std::begin(container), std::end(container), &values_[0]); - } - - // Fills the array with the given value. - void Fill(const T& value) { - std::fill(&values_[0], &values_[0] + num_elements(), value); - } + int64 n4() const { return this->dim(3); } + int64 n3() const { return this->dim(2); } + int64 n2() const { return this->dim(1); } + int64 n1() const { return this->dim(0); } - // Fills the array with iota. - void FillIota(const T& value) { - std::iota(&values_[0], &values_[0] + num_elements(), value); - } - - // Fills the array with random variable with a deviation of value and a mean - // of mean. - void FillRandom(const T& value, const double mean = 0.0, - const int seed = 12345) { - std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(value)); - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); - } - } - - // Fills values with the sequence i*multiplier for i=0,1,... - void FillWithMultiples(float multiplier) { - for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = i * multiplier; - } - } - - // Invokes a callback with the (indices, value_ptr) for each cell in the 4D - // array. - void Each(std::function, T*)> f) { - for (int64 plane = 0; plane < planes(); ++plane) { - for (int64 depth = 0; depth < this->depth(); ++depth) { - for (int64 height = 0; height < this->height(); ++height) { - for (int64 width = 0; width < this->width(); ++width) { - auto& value = (*this)(plane, depth, height, width); - f({plane, depth, height, width}, &value); - } - } - } - } - } - - // Invokes a callback with the (indices, value) for each cell in the 4D array. - void Each( - std::function, T)> f) const { - // We const_cast to be able to use the common non-const implementation, - // but prevent modification of the data by passing it by-value to the - // caller. - const_cast(this)->Each( - [&f](tensorflow::gtl::ArraySlice indices, T* value) { - f(indices, *value); - }); - } + int64 width() const { return this->dim(3); } + int64 height() const { return this->dim(2); } + int64 depth() const { return this->dim(1); } + int64 planes() const { return this->dim(0); } // Fills all of the {p,z} with the array provided, which specifies {y,x}. void FillWithYX(const Array2D& value) { @@ -267,38 +142,6 @@ class Array4D { } } } - - // Returns a string representation of the 4D array suitable for debugging. - string ToString() const { - std::vector pieces = { - tensorflow::strings::Printf("p=%lld,z=%lld,y=%lld,x=%lld {\n", planes(), - depth(), height(), width())}; - for (int64 plane = 0; plane < planes_; ++plane) { - pieces.push_back(" {\n"); - for (int64 depth = 0; depth < depth_; ++depth) { - pieces.push_back(" {\n"); - for (int64 height = 0; height < height_; ++height) { - pieces.push_back(" {"); - for (int64 width = 0; width < width_; ++width) { - pieces.push_back(tensorflow::strings::StrCat( - (*this)(plane, depth, height, width), ", ")); - } - pieces.push_back("},\n"); - } - pieces.push_back(" },\n"); - } - pieces.push_back(" },\n"); - } - pieces.push_back("}"); - return tensorflow::str_util::Join(pieces, ""); - } - - private: - int64 planes_; - int64 depth_; - int64 height_; - int64 width_; - std::unique_ptr values_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/array_test.cc b/tensorflow/compiler/xla/array_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..093784f541b3bd18f4a1fc1b665cd0d17a892f28 --- /dev/null +++ b/tensorflow/compiler/xla/array_test.cc @@ -0,0 +1,145 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/array.h" + +#include + +#include "tensorflow/compiler/xla/test.h" + +namespace xla { +namespace { + +TEST(ArrayTest, UninitializedDimsCtor) { + Array uninit({2, 3}); + EXPECT_EQ(uninit.num_dimensions(), 2); + EXPECT_EQ(uninit.dim(0), 2); + EXPECT_EQ(uninit.dim(1), 3); + EXPECT_EQ(uninit.num_elements(), 6); +} + +TEST(ArrayTest, FillCtor) { + Array fullof7({1, 2, 3}, 7); + + EXPECT_EQ(fullof7.dim(0), 1); + EXPECT_EQ(fullof7.dim(1), 2); + EXPECT_EQ(fullof7.dim(2), 3); + + for (int64 n0 = 0; n0 < fullof7.dim(0); ++n0) { + for (int64 n1 = 0; n1 < fullof7.dim(1); ++n1) { + for (int64 n2 = 0; n2 < fullof7.dim(2); ++n2) { + EXPECT_EQ(fullof7(n0, n1, n2), 7); + } + } + } +} + +TEST(ArrayTest, InitializerListCtor) { + Array arr({{1, 2, 3}, {4, 5, 6}}); + + EXPECT_EQ(arr.dim(0), 2); + EXPECT_EQ(arr.dim(1), 3); + + EXPECT_EQ(arr(0, 0), 1); + EXPECT_EQ(arr(0, 1), 2); + EXPECT_EQ(arr(0, 2), 3); + EXPECT_EQ(arr(1, 0), 4); + EXPECT_EQ(arr(1, 1), 5); + EXPECT_EQ(arr(1, 2), 6); +} + +TEST(ArrayTest, IndexingReadWrite) { + Array arr({2, 3}); + + EXPECT_EQ(arr(1, 1), 0); + EXPECT_EQ(arr(1, 2), 0); + arr(1, 1) = 51; + arr(1, 2) = 61; + EXPECT_EQ(arr(1, 1), 51); + EXPECT_EQ(arr(1, 2), 61); +} + +TEST(ArrayTest, IndexingReadWriteBool) { + Array arr{{false, true, false}, {false, true, false}}; + + EXPECT_EQ(arr(0, 1), true); + EXPECT_EQ(arr(0, 2), false); + arr(0, 1) = false; + arr(0, 2) = true; + EXPECT_EQ(arr(0, 1), false); + EXPECT_EQ(arr(0, 2), true); +} + +TEST(ArrayTest, Fill) { + Array fullof7({2, 3}, 7); + for (int64 n1 = 0; n1 < fullof7.dim(0); ++n1) { + for (int64 n2 = 0; n2 < fullof7.dim(1); ++n2) { + EXPECT_EQ(fullof7(n1, n2), 7); + } + } + + fullof7.Fill(11); + for (int64 n1 = 0; n1 < fullof7.dim(0); ++n1) { + for (int64 n2 = 0; n2 < fullof7.dim(1); ++n2) { + EXPECT_EQ(fullof7(n1, n2), 11); + } + } +} + +TEST(ArrayTest, DataPointer) { + Array arr{{1, 2, 3}, {4, 5, 6}}; + EXPECT_EQ(arr.data()[0], 1); +} + +TEST(ArrayTest, Stringification1D) { + Array arr({2}, 1); + const string expected = R"([1, 1])"; + EXPECT_EQ(expected, arr.ToString()); +} + +TEST(ArrayTest, Stringification2D) { + Array arr({2, 3}, 7); + const string expected = "[[7, 7, 7],\n [7, 7, 7]]"; + EXPECT_EQ(expected, arr.ToString()); +} + +TEST(ArrayTest, Stringification3D) { + Array arr({2, 3, 4}, 5); + const string expected = R"([[[5, 5, 5, 5], + [5, 5, 5, 5], + [5, 5, 5, 5]], + [[5, 5, 5, 5], + [5, 5, 5, 5], + [5, 5, 5, 5]]])"; + EXPECT_EQ(expected, arr.ToString()); +} + +TEST(ArrayTest, Each) { + Array arr({2, 3, 4}); + arr.FillWithMultiples(1); + + int64 each_count = 0, each_sum = 0; + arr.Each([&](tensorflow::gtl::ArraySlice idx, int cell) { + int64 lin_idx = idx[0] * 12 + idx[1] * 4 + idx[2]; + EXPECT_EQ(lin_idx, cell); + each_count++; + each_sum += cell; + }); + EXPECT_EQ(arr.num_elements(), each_count); + EXPECT_EQ(arr.num_elements() * (arr.num_elements() - 1) / 2, each_sum); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD index b6126981431dc9a3520b6c96321c453bc955e7c0..f953407a567b91fdf6ae727d6982a2a778c5873e 100644 --- a/tensorflow/compiler/xla/client/BUILD +++ b/tensorflow/compiler/xla/client/BUILD @@ -170,6 +170,7 @@ cc_library( ":computation", ":global_data", ":padding", + "//tensorflow/compiler/xla:array", "//tensorflow/compiler/xla:array2d", "//tensorflow/compiler/xla:array3d", "//tensorflow/compiler/xla:array4d", diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc index dcbdb3525e8d4f397a9934f2658c7cc72b9144da..24774c4c2a385d9aabd22a550bd8be3acf409d85 100644 --- a/tensorflow/compiler/xla/client/computation_builder.cc +++ b/tensorflow/compiler/xla/client/computation_builder.cc @@ -663,7 +663,7 @@ bool ComputationBuilder::VerifyConvolution( return false; } int num_dims = ShapeUtil::Rank(lhs_shape); - if (num_dims < 3) { + if (num_dims < 2) { NoteError(InvalidArgument( "Convolution expects argument arrays with >= 3 dimensions. " "Got: %s and %s", @@ -913,6 +913,17 @@ ComputationDataHandle ComputationBuilder::CustomCall( return ParseOpResponse(s, &response); } +ComputationDataHandle ComputationBuilder::Complex( + const ComputationDataHandle& real, const ComputationDataHandle& imag, + tensorflow::gtl::ArraySlice broadcast_dimensions) { + return BinaryOp(BINOP_COMPLEX, real, imag, broadcast_dimensions); +} + +ComputationDataHandle ComputationBuilder::Conj( + const ComputationDataHandle& operand) { + return Complex(Real(operand), Neg(Imag(operand))); +} + ComputationDataHandle ComputationBuilder::Add( const ComputationDataHandle& lhs, const ComputationDataHandle& rhs, tensorflow::gtl::ArraySlice broadcast_dimensions) { @@ -995,6 +1006,12 @@ ComputationDataHandle ComputationBuilder::Abs( return UnaryOp(UNOP_ABS, operand); } +ComputationDataHandle ComputationBuilder::Atan2( + const ComputationDataHandle& y, const ComputationDataHandle& x, + tensorflow::gtl::ArraySlice broadcast_dimensions) { + return BinaryOp(BINOP_ATAN2, y, x, broadcast_dimensions); +} + ComputationDataHandle ComputationBuilder::Exp( const ComputationDataHandle& operand) { return UnaryOp(UNOP_EXP, operand); @@ -1040,6 +1057,16 @@ ComputationDataHandle ComputationBuilder::Tanh( return UnaryOp(UNOP_TANH, operand); } +ComputationDataHandle ComputationBuilder::Real( + const ComputationDataHandle& operand) { + return UnaryOp(UNOP_REAL, operand); +} + +ComputationDataHandle ComputationBuilder::Imag( + const ComputationDataHandle& operand) { + return UnaryOp(UNOP_IMAG, operand); +} + ComputationDataHandle ComputationBuilder::IsFinite( const ComputationDataHandle& operand) { return UnaryOp(UNOP_IS_FINITE, operand); @@ -1767,14 +1794,9 @@ StatusOr ComputationBuilder::Build() { void ComputationBuilder::AddCommonFieldsToOpRequest(OpRequest* request) const { *request->mutable_metadata() = metadata_; - *request->mutable_device_assignment() = device_assignment_; -} - -void ComputationBuilder::ClearDeviceAssignment() { device_assignment_.Clear(); } - -void ComputationBuilder::SetDeviceAssignment( - const OpDeviceAssignment& assignment) { - device_assignment_ = assignment; + if (sharding_) { + *request->mutable_sharding() = *sharding_; + } } /* static */ ConvolutionDimensionNumbers diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h index cdd9c8847f56e25bcb807a9cf0631e72bf4355ee..d282174947970ab13a8b29ba4212d56ceb0c572a 100644 --- a/tensorflow/compiler/xla/client/computation_builder.h +++ b/tensorflow/compiler/xla/client/computation_builder.h @@ -22,6 +22,7 @@ limitations under the License. #include #include +#include "tensorflow/compiler/xla/array.h" #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/array3d.h" #include "tensorflow/compiler/xla/array4d.h" @@ -42,6 +43,58 @@ limitations under the License. namespace xla { +class ShardingBuilder { + public: + // A shaped array used to describe the assignment of tiles to devices. + using TileAssignment = Array; + + // Creates a replicated sharding - replicate a tensor on every device. + static OpSharding Replicate() { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_REPLICATED); + return result; + } + // Creates a sharding that assigns a tensor to just one device. + static OpSharding AssignDevice(int device) { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + result.add_tile_assignment_dimensions(1); + result.add_tile_assignment_devices(device); + return result; + } + // Creates a tiled sharding with the given tile shape and assignment of tiles + // to devices. + static OpSharding Tile(Shape tile_shape, + const TileAssignment& tile_assignment) { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_OTHER); + for (int64 dim : tile_assignment.dimensions()) { + result.add_tile_assignment_dimensions(dim); + } + for (uint32 device : tile_assignment) { + result.add_tile_assignment_devices(device); + } + return result; + } + // Creates a sharding in one dimension, with the given tile shape which must + // be rank 1 and using devices 0..num_tiles. + static OpSharding Tile1D(Shape tile_shape, int64 num_tiles) { + OpSharding result; + result.set_type(OpSharding::Type::OpSharding_Type_OTHER); + + CHECK_EQ(ShapeUtil::Rank(tile_shape), 1); + std::vector dimensions(1, num_tiles); + auto& tile_dimension = (*tile_shape.mutable_dimensions())[0]; + tile_dimension = CeilOfRatio(static_cast(tile_dimension), num_tiles); + *result.mutable_tile_shape() = tile_shape; + result.add_tile_assignment_dimensions(num_tiles); + for (int64 i = 0; i < num_tiles; ++i) { + result.add_tile_assignment_devices(i); + } + return result; + } +}; + // Wraps an XLA client with a convenient interface for building up // computations. Any errors encountered in building up the computation are // deferred from being handled until Build() is called. @@ -78,11 +131,11 @@ class ComputationBuilder { // Sets an OpDeviceAssignment that will be attached to all instructions // until cleared. - void SetDeviceAssignment(const OpDeviceAssignment& assignment); + void SetSharding(const OpSharding& sharding) { sharding_ = sharding; } // Clears the device assignment. Ops will be placed according to the default // placement policy. - void ClearDeviceAssignment(); + void ClearSharding() { sharding_ = tensorflow::gtl::nullopt; } // Sets the builder to a mode where it will die immediately when an error is // encountered, rather than producing it in a deferred fashion when Build() is @@ -138,6 +191,11 @@ class ComputationBuilder { ComputationDataHandle ConstantR2( std::initializer_list> values); template + ComputationDataHandle ConstantFromArrayWithLayout( + const Array& values, const Layout& layout); + template + ComputationDataHandle ConstantFromArray(const Array& values); + template ComputationDataHandle ConstantR2FromArray2DWithLayout( const Array2D& values, const Layout& layout); template @@ -426,6 +484,14 @@ class ComputationBuilder { // of the operands is a scalar, or an explicit broadcast dimension is given // (see g3doc for more details). + // Enqueues a complex compose instruction onto the computation. + ComputationDataHandle Complex( + const ComputationDataHandle& real, const ComputationDataHandle& imag, + tensorflow::gtl::ArraySlice broadcast_dimensions = {}); + + // Enqueues a complex conjugate instruction onto the computation. + ComputationDataHandle Conj(const ComputationDataHandle& operand); + // Enqueues an add instruction onto the computation. ComputationDataHandle Add( const ComputationDataHandle& lhs, const ComputationDataHandle& rhs, @@ -537,6 +603,11 @@ class ComputationBuilder { // Enqueues an abs instruction onto the computation. ComputationDataHandle Abs(const ComputationDataHandle& operand); + // Enqueues a atan2 instruction onto the computation. + ComputationDataHandle Atan2( + const ComputationDataHandle& y, const ComputationDataHandle& x, + tensorflow::gtl::ArraySlice broadcast_dimensions = {}); + // Enqueues an exp instruction onto the computation. ComputationDataHandle Exp(const ComputationDataHandle& operand); @@ -565,6 +636,12 @@ class ComputationBuilder { // Enqueues a tanh instruction onto the computation. ComputationDataHandle Tanh(const ComputationDataHandle& operand); + // Enqueues a real-part instruction onto the computation. + ComputationDataHandle Real(const ComputationDataHandle& operand); + + // Enqueues an imaginary-part instruction onto the computation. + ComputationDataHandle Imag(const ComputationDataHandle& operand); + // Enqueues a float32 sqrt instruction onto the computation. // (float32 is specified as there is an implicit float32 0.5f constant // exponent). @@ -870,8 +947,9 @@ class ComputationBuilder { // throughout the TensorFlow op kernel implementations). OpMetadata metadata_; - // Device assignment for the operator. - OpDeviceAssignment device_assignment_; + // Sharding for this operator. This is structured as a "model"-like operation, + // in order to simplify client code, similar to metadata_. + tensorflow::gtl::optional sharding_; TF_DISALLOW_COPY_AND_ASSIGN(ComputationBuilder); }; @@ -910,48 +988,54 @@ ComputationDataHandle ComputationBuilder::ConstantR2( } template -ComputationDataHandle ComputationBuilder::ConstantR2FromArray2DWithLayout( - const Array2D& values, const Layout& layout) { +ComputationDataHandle ComputationBuilder::ConstantFromArrayWithLayout( + const Array& values, const Layout& layout) { return ConstantOp([&values, &layout](Literal* literal) { - literal->PopulateR2FromArray2DWithLayout(values, layout); + literal->PopulateFromArrayWithLayout(values, layout); }); } +template +ComputationDataHandle ComputationBuilder::ConstantFromArray( + const Array& values) { + return ConstantOp( + [&values](Literal* literal) { literal->PopulateFromArray(values); }); +} + +template +ComputationDataHandle ComputationBuilder::ConstantR2FromArray2DWithLayout( + const Array2D& values, const Layout& layout) { + return ConstantFromArrayWithLayout(values, layout); +} + template ComputationDataHandle ComputationBuilder::ConstantR2FromArray2D( const Array2D& values) { - return ConstantOp( - [&values](Literal* literal) { literal->PopulateR2FromArray2D(values); }); + return ConstantFromArray(values); } template ComputationDataHandle ComputationBuilder::ConstantR3FromArray3DWithLayout( const Array3D& values, const Layout& layout) { - return ConstantOp([&values, &layout](Literal* literal) { - literal->PopulateR3FromArray3DWithLayout(values, layout); - }); + return ConstantFromArrayWithLayout(values, layout); } template ComputationDataHandle ComputationBuilder::ConstantR3FromArray3D( const Array3D& values) { - return ConstantOp( - [&values](Literal* literal) { literal->PopulateR3FromArray3D(values); }); + return ConstantFromArray(values); } template ComputationDataHandle ComputationBuilder::ConstantR4FromArray4DWithLayout( const Array4D& values, const Layout& layout) { - return ConstantOp([&values, &layout](Literal* literal) { - literal->PopulateR4FromArray4DWithLayout(values, layout); - }); + return ConstantFromArrayWithLayout(values, layout); } template ComputationDataHandle ComputationBuilder::ConstantR4FromArray4D( const Array4D& values) { - return ConstantOp( - [&values](Literal* literal) { literal->PopulateR4FromArray4D(values); }); + return ConstantFromArray(values); } } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/testing.cc b/tensorflow/compiler/xla/client/lib/testing.cc index 482d53cf330f152f496b77233714f93991fef6f0..e6645e4941bd04c658b67117bb689f6fdef7dfc1 100644 --- a/tensorflow/compiler/xla/client/lib/testing.cc +++ b/tensorflow/compiler/xla/client/lib/testing.cc @@ -79,6 +79,24 @@ StatusOr> MakeFakeLiteral(const Shape& shape) { })); break; } + case S64: { + std::uniform_int_distribution generator( + std::numeric_limits::lowest(), + std::numeric_limits::max()); + TF_CHECK_OK(literal->Populate( + [&](tensorflow::gtl::ArraySlice /*indices*/) { + return generator(engine); + })); + break; + } + case PRED: { + std::uniform_int_distribution generator(0, 1); + TF_CHECK_OK(literal->Populate( + [&](tensorflow::gtl::ArraySlice /*indices*/) { + return generator(engine); + })); + break; + } default: return Unimplemented("Unsupported type for fake literal generation: %s", ShapeUtil::HumanString(shape).c_str()); diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc index c885b815ebef60bbabfdbd97642d0be9bbbf49e8..15c744ecd349e91dc703bec5708d78a896f132c3 100644 --- a/tensorflow/compiler/xla/client/local_client.cc +++ b/tensorflow/compiler/xla/client/local_client.cc @@ -175,10 +175,15 @@ StatusOr> LocalExecutable::Run( TF_RETURN_IF_ERROR(ValidateExecutionOptions(arguments, options, *backend_)); ExecutableRunOptions actual_options = options; + + Backend::StreamPtr stream; if (options.stream() == nullptr) { + // NB! The lifetime of `stream` needs to match the lifetime of + // `actual_options` (otherwise we will end up using a returned stream in + // ExecuteOnStreamWrapper), which is why it isn't declared in the inner "if" + // scope. TF_ASSIGN_OR_RETURN( - Backend::StreamPtr stream, - BorrowStreamForDevice(options.device_ordinal(), backend_)); + stream, BorrowStreamForDevice(options.device_ordinal(), backend_)); actual_options.set_stream(stream.get()); } if (options.allocator() == nullptr) { diff --git a/tensorflow/compiler/xla/layout_util.cc b/tensorflow/compiler/xla/layout_util.cc index 011fc3c194e0eb9ebd6b9e42571deddaf25c09ff..5c2cc2a7a99cc51ded3d98c9dd5903e4b3078548 100644 --- a/tensorflow/compiler/xla/layout_util.cc +++ b/tensorflow/compiler/xla/layout_util.cc @@ -83,6 +83,10 @@ Layout CreateDefaultLayoutForRank(int64 rank) { return CreateDefaultLayoutForRank(shape.dimensions_size()); } +/* static */ Layout LayoutUtil::GetDefaultLayoutForRank(int64 rank) { + return CreateDefaultLayoutForRank(rank); +} + /* static */ Layout LayoutUtil::GetDefaultLayoutForR2() { return CreateDefaultLayoutForRank(2); } diff --git a/tensorflow/compiler/xla/layout_util.h b/tensorflow/compiler/xla/layout_util.h index 5de0a653f66688ac75fc377c18ff93012314abdd..bc42e222292933be35e82d1fe50802e8830d16b3 100644 --- a/tensorflow/compiler/xla/layout_util.h +++ b/tensorflow/compiler/xla/layout_util.h @@ -40,6 +40,7 @@ class LayoutUtil { static Layout GetDefaultLayoutForShape(const Shape& shape); // Helper functions that create default layouts for various ranks. + static Layout GetDefaultLayoutForRank(int64 rank); static Layout GetDefaultLayoutForR2(); static Layout GetDefaultLayoutForR3(); static Layout GetDefaultLayoutForR4(); diff --git a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc index 8892bfbe929d168c602af24cfbb507256dc05328..f2cdd9669c727bb778fce495ede0faaf2d9a923d 100644 --- a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc +++ b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc @@ -206,9 +206,9 @@ void AllocateFlags() { flag_values->xla_gpu_disable_multi_streaming(), "If true, multi-streaming in the GPU backend is disabled."), tensorflow::Flag( - "xla_dump_debug_json_to", - flag_values->mutable_xla_dump_debug_json_to(), - "Dump compilation artifacts as JSON into this directory."), + "xla_dump_hlo_proto_to", + flag_values->mutable_xla_dump_hlo_proto_to(), + "Dump compilation artifacts as proto binary into this directory."), tensorflow::Flag( "xla_test_all_output_layouts", bool_setter_for(&DebugOptions::set_xla_test_all_output_layouts), diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc index 79e40c12625c41b7234542381d0ca528be7eaed4..8fc8644a60ef62d7ba5e7f0cc11253742395f09b 100644 --- a/tensorflow/compiler/xla/literal_util.cc +++ b/tensorflow/compiler/xla/literal_util.cc @@ -173,6 +173,8 @@ Status Literal::Copy(const Literal& src_literal, return CopyRange(src_literal, src_base, dest_base, copy_size); case F64: return CopyRange(src_literal, src_base, dest_base, copy_size); + case C64: + return CopyRange(src_literal, src_base, dest_base, copy_size); case PRED: return CopyRange(src_literal, src_base, dest_base, copy_size); default: @@ -202,6 +204,8 @@ Status Literal::Copy(const Literal& src_literal, return *Literal::CreateR0(0); case F64: return *Literal::CreateR0(0); + case C64: + return *Literal::CreateR0(0); case PRED: return *Literal::CreateR0(false); case S16: @@ -234,6 +238,8 @@ Status Literal::Copy(const Literal& src_literal, return *Literal::CreateR0(1); case F64: return *Literal::CreateR0(1); + case C64: + return *Literal::CreateR0(1); case PRED: return *Literal::CreateR0(true); case S16: @@ -269,6 +275,8 @@ Status Literal::Copy(const Literal& src_literal, case F64: return *Literal::CreateR0( -std::numeric_limits::infinity()); + case C64: + LOG(FATAL) << "C64 element type has no minimum value"; case PRED: return *Literal::CreateR0(false); case S16: @@ -522,6 +530,10 @@ string Literal::GetAsString( return tensorflow::strings::StrCat(Get(multi_index)); case F64: return tensorflow::strings::StrCat(Get(multi_index)); + case C64: { + complex64 c = Get(multi_index); + return tensorflow::strings::StrCat("(", c.real(), ", ", c.imag(), ")"); + } case F16: return tensorflow::strings::StrCat(Get(multi_index)); default: @@ -716,6 +728,8 @@ void* Literal::MutableInternalData() { return reinterpret_cast(f32s_.data()); case F64: return reinterpret_cast(f64s_.data()); + case C64: + return reinterpret_cast(c64s_.data()); case F16: return reinterpret_cast(f16s_.data()); default: @@ -754,6 +768,9 @@ void Literal::Reserve(int64 num_elements) { case F64: Resize(num_elements, 0); break; + case C64: + Resize(num_elements, 0); + break; case F16: Resize(num_elements, static_cast(0.0f)); break; @@ -790,6 +807,9 @@ tensorflow::Status Literal::ValidateLiteral() const { case F64: actual = f64s_size(); break; + case C64: + actual = c64s_size(); + break; case F16: actual = f16s().size() / sizeof(half); break; @@ -843,6 +863,26 @@ std::unique_ptr ConvertBetweenNativeTypes(const Literal& src_literal) { return result_literal; } +template +std::unique_ptr ConvertToC64(const Literal& src_literal) { + auto result_literal = MakeUnique(); + Shape* result_shape = result_literal->mutable_shape(); + *result_shape = src_literal.shape(); + result_shape->set_element_type(C64); + result_literal->Reserve(ShapeUtil::ElementsIn(*result_shape)); + using NativeSrcT = + typename primitive_util::PrimitiveTypeToNative::type; + tensorflow::gtl::ArraySlice src_data = + src_literal.GetArraySlice(); + tensorflow::gtl::MutableArraySlice dest_data = + result_literal->GetMutableArraySlice(); + int64 num_elements = ShapeUtil::ElementsIn(src_literal.shape()); + for (int64 i = 0; i < num_elements; ++i) { + dest_data[i] = complex64(static_cast(src_data[i]), 0); + } + return result_literal; +} + template std::unique_ptr ConvertIfTypesMatch(const Literal& src_literal) { CHECK_EQ(primitive_src_type, src_literal.shape().element_type()); @@ -870,6 +910,8 @@ StatusOr> ConvertIfDestTypeMatches( CONVERT_IF_TYPES_MATCH(F32) CONVERT_IF_TYPES_MATCH(F64) #undef CONVERT_IF_TYPES_MATCH + case C64: + return ConvertToC64(src_literal); // Other types are not yet supported. default: return InvalidArgument( @@ -966,6 +1008,8 @@ bool Literal::operator==(const Literal& other) const { return EqualElements(*this, other, 0, &multi_index); case F16: return EqualElements(*this, other, 0, &multi_index); + case C64: + return EqualElements(*this, other, 0, &multi_index); default: LOG(FATAL) << "Unimplemented: Literal::Equal for type " << PrimitiveType_Name(shape().element_type()); @@ -1065,6 +1109,12 @@ tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice() { values->size()); } +template <> +tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice() { + auto values = mutable_c64s(); + return {values->data(), values->size()}; +} + template <> tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice() { // TODO - there is an endianess problem here. fix it, or wait for uint16 @@ -1144,6 +1194,13 @@ tensorflow::gtl::ArraySlice Literal::GetArraySlice() const { f16s().size() / sizeof(half)); } +template <> +tensorflow::gtl::ArraySlice Literal::GetArraySlice() + const { + CHECK_EQ(shape().element_type(), C64); + return c64s(); +} + template static bool AllElementsEqualValue(const Literal& literal, NativeT value) { for (int64 i = 0; i < ShapeUtil::ElementsIn(literal.shape()); ++i) { @@ -1211,6 +1268,15 @@ bool Literal::IsAllFloat(float value) const { } } +bool Literal::IsAllComplex(complex64 value) const { + switch (shape().element_type()) { + case C64: + return AllElementsEqualValue(*this, value); + default: + return false; + } +} + bool Literal::IsZero(tensorflow::gtl::ArraySlice indices) const { switch (shape().element_type()) { case U8: @@ -1229,6 +1295,8 @@ bool Literal::IsZero(tensorflow::gtl::ArraySlice indices) const { return Get(indices) == 0.0f; case F64: return Get(indices) == 0.0; + case C64: + return Get(indices) == complex64(0.0f, 0.0f); case F16: return Get(indices) == static_cast(0.0f); case PRED: @@ -1298,12 +1366,27 @@ void Literal::Resize(int64 num_elements, half value) { mutable_f16s()->resize(num_elements, value); } +template <> +void Literal::Resize(int64 num_elements, complex64 value) { + CHECK_EQ(ShapeUtil::ElementsIn(shape()), num_elements); + mutable_c64s()->resize(num_elements, value); +} + template -static void CopyToRepeatedField(RepeatedFieldT* dest, - const std::vector& src) { +void CopyToRepeatedField(RepeatedFieldT* dest, + const std::vector& src) { *dest = RepeatedFieldT(src.begin(), src.end()); } +template <> +void CopyToRepeatedField, complex64>( + tensorflow::protobuf::RepeatedField* dest, + const std::vector& src) { + *dest = tensorflow::protobuf::RepeatedField( + reinterpret_cast(src.data()), + reinterpret_cast(src.data()) + src.size() * 2); +} + LiteralProto Literal::ToProto() const { LiteralProto proto; proto.Clear(); @@ -1338,6 +1421,9 @@ LiteralProto Literal::ToProto() const { case F64: CopyToRepeatedField(proto.mutable_f64s(), f64s()); break; + case C64: + CopyToRepeatedField(proto.mutable_c64s(), c64s()); + break; case TUPLE: for (const auto& tuple : tuple_literals()) { *proto.add_tuple_literals() = tuple.ToProto(); @@ -1351,11 +1437,21 @@ LiteralProto Literal::ToProto() const { } template -static void CopyFromRepeatedField(std::vector* dest, - const RepeatedFieldT& src) { +void CopyFromRepeatedField(std::vector* dest, + const RepeatedFieldT& src) { *dest = std::vector(src.begin(), src.end()); } +template <> +void CopyFromRepeatedField, + complex64>( + std::vector* dest, + const tensorflow::protobuf::RepeatedField& src) { + *dest = std::vector( + reinterpret_cast(src.data()), + reinterpret_cast(src.data()) + src.size() / 2); +} + void Literal::CopyFromProto(const LiteralProto& literal_proto) { if (!literal_proto.has_shape()) { return; @@ -1394,6 +1490,9 @@ void Literal::CopyFromProto(const LiteralProto& literal_proto) { case F64: CopyFromRepeatedField(mutable_f64s(), literal_proto.f64s()); break; + case C64: + CopyFromRepeatedField(mutable_c64s(), literal_proto.c64s()); + break; case TUPLE: for (const auto& proto : literal_proto.tuple_literals()) { mutable_tuple_literals()->push_back(Literal(proto)); diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h index e8cee732d4cf5629c1e2b9c98d1f1bbe1e29a122..a1e288829f22835f94c6e3c041796f84d995211c 100644 --- a/tensorflow/compiler/xla/literal_util.h +++ b/tensorflow/compiler/xla/literal_util.h @@ -159,6 +159,10 @@ class Literal { const std::vector& f64s() const { return f64s_; } std::vector* mutable_f64s() { return &f64s_; } + int c64s_size() const { return c64s().size(); } + const std::vector& c64s() const { return c64s_; } + std::vector* mutable_c64s() { return &c64s_; } + int tuple_literals_size() const { return tuple_literals().size(); } const Literal& tuple_literals(int i) const { return tuple_literals_[i]; } Literal* add_tuple_literals() { @@ -334,6 +338,11 @@ class Literal { // WithLayout use the default XLA layout for the literal's linear // representation in memory. template + static std::unique_ptr CreateFromArray(const Array& values); + template + static std::unique_ptr CreateFromArrayWithLayout( + const Array& values, const Layout& layout); + template static std::unique_ptr CreateR2FromArray2D( const Array2D& values); template @@ -481,6 +490,11 @@ class Literal { std::initializer_list> values, const Layout& layout); template + void PopulateFromArray(const Array& values); + template + void PopulateFromArrayWithLayout(const Array& values, + const Layout& layout); + template void PopulateR2FromArray2D(const Array2D& values); template void PopulateR2FromArray2DWithLayout(const Array2D& values, @@ -550,6 +564,17 @@ class Literal { // e.g. -0.5. bool IsAllFloat(float value) const; + // Like IsAll(const Literal&, int8), except we check whether the literal is + // equal to a particular complex number. + // + // If the literal is not a complex value, this always returns false. + // + // This casts value to the type of literal, then compares using ==. The usual + // admonishments about floating-point equality checks apply. We expect you to + // use this to check for complex values that can be expressed precisely as + // float pairs e.g. (-0.5, 1.0). + bool IsAllComplex(complex64 value) const; + // Returns whether this literal is zero at the specified index. This literal // must be an array. bool IsZero(tensorflow::gtl::ArraySlice indices) const; @@ -600,6 +625,7 @@ class Literal { std::vector f16s_; std::vector f32s_; std::vector f64s_; + std::vector c64s_; std::vector tuple_literals_; }; @@ -648,6 +674,10 @@ tensorflow::gtl::ArraySlice Literal::GetArraySlice() const; template <> tensorflow::gtl::ArraySlice Literal::GetArraySlice() const; +template <> +tensorflow::gtl::ArraySlice Literal::GetArraySlice() + const; + template <> tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); @@ -684,6 +714,9 @@ tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); template <> tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); +template <> +tensorflow::gtl::MutableArraySlice Literal::GetMutableArraySlice(); + template <> void Literal::Resize(int64 num_elements, bool value); @@ -714,6 +747,9 @@ void Literal::Resize(int64 num_elements, double value); template <> void Literal::Resize(int64 num_elements, half value); +template <> +void Literal::Resize(int64 num_elements, complex64 value); + template /* static */ std::unique_ptr Literal::CreateR0(NativeT value) { auto literal = MakeUnique(); @@ -816,33 +852,42 @@ template } template -/* static */ std::unique_ptr Literal::CreateR2FromArray2DWithLayout( - const Array2D& values, const Layout& layout) { +/* static */ std::unique_ptr Literal::CreateFromArrayWithLayout( + const Array& values, const Layout& layout) { auto literal = MakeUnique(); - literal->PopulateR2FromArray2DWithLayout(values, layout); + literal->PopulateFromArrayWithLayout(values, layout); return literal; } +template +/* static */ std::unique_ptr Literal::CreateFromArray( + const Array& values) { + return CreateFromArrayWithLayout( + values, LayoutUtil::GetDefaultLayoutForRank(values.num_dimensions())); +} + +template +/* static */ std::unique_ptr Literal::CreateR2FromArray2DWithLayout( + const Array2D& values, const Layout& layout) { + return CreateFromArrayWithLayout(values, layout); +} + template /* static */ std::unique_ptr Literal::CreateR2FromArray2D( const Array2D& values) { - return CreateR2FromArray2DWithLayout(values, - LayoutUtil::GetDefaultLayoutForR2()); + return CreateFromArray(values); } template /* static */ std::unique_ptr Literal::CreateR3FromArray3DWithLayout( const Array3D& values, const Layout& layout) { - auto literal = MakeUnique(); - literal->PopulateR3FromArray3DWithLayout(values, layout); - return literal; + return CreateFromArrayWithLayout(values, layout); } template /* static */ std::unique_ptr Literal::CreateR3FromArray3D( const Array3D& values) { - return CreateR3FromArray3DWithLayout(values, - LayoutUtil::GetDefaultLayoutForR3()); + return CreateFromArray(values); } template @@ -901,16 +946,13 @@ template template /* static */ std::unique_ptr Literal::CreateR4FromArray4D( const Array4D& values) { - return CreateR4FromArray4DWithLayout(values, - LayoutUtil::GetDefaultLayoutForR4()); + return CreateFromArray(values); } template /* static */ std::unique_ptr Literal::CreateR4FromArray4DWithLayout( const Array4D& values, const Layout& layout) { - auto literal = MakeUnique(); - literal->PopulateR4FromArray4DWithLayout(values, layout); - return literal; + return CreateFromArrayWithLayout(values, layout); } template @@ -1070,82 +1112,53 @@ void Literal::PopulateR2( } template -void Literal::PopulateR2FromArray2DWithLayout(const Array2D& values, - const Layout& layout) { +void Literal::PopulateFromArrayWithLayout(const Array& values, + const Layout& layout) { *mutable_shape() = ShapeUtil::MakeShapeWithLayout( - primitive_util::NativeToPrimitiveType(), - {values.height(), values.width()}, AsInt64Slice(layout.minor_to_major())); + primitive_util::NativeToPrimitiveType(), values.dimensions(), + AsInt64Slice(layout.minor_to_major())); + Reserve(values.num_elements()); + values.Each([this](tensorflow::gtl::ArraySlice indices, + NativeT value) { this->Set(indices, value); }); +} - const int64 dim1_size = values.width(); - const int64 dim0_size = values.height(); - CHECK_EQ(dim0_size, shape().dimensions(0)); - CHECK_EQ(dim1_size, shape().dimensions(1)); - Reserve(dim1_size * dim0_size); - for (int64 dim0 = 0; dim0 < dim0_size; ++dim0) { - for (int64 dim1 = 0; dim1 < dim1_size; ++dim1) { - Set({dim0, dim1}, values(dim0, dim1)); - } - } +template +void Literal::PopulateFromArray(const Array& values) { + PopulateFromArrayWithLayout( + values, LayoutUtil::GetDefaultLayoutForRank(values.num_dimensions())); +} + +template +void Literal::PopulateR2FromArray2DWithLayout(const Array2D& values, + const Layout& layout) { + PopulateFromArrayWithLayout(values, layout); } template void Literal::PopulateR2FromArray2D(const Array2D& values) { - PopulateR2FromArray2DWithLayout(values, LayoutUtil::GetDefaultLayoutForR2()); + PopulateFromArray(values); } template void Literal::PopulateR3FromArray3DWithLayout(const Array3D& values, const Layout& layout) { - *mutable_shape() = ShapeUtil::MakeShapeWithLayout( - primitive_util::NativeToPrimitiveType(), - {values.n1(), values.n2(), values.n3()}, - AsInt64Slice(layout.minor_to_major())); - - CHECK_EQ(values.n1(), shape().dimensions(0)); - CHECK_EQ(values.n2(), shape().dimensions(1)); - CHECK_EQ(values.n3(), shape().dimensions(2)); - Reserve(values.n1() * values.n2() * values.n3()); - for (int64 dim0 = 0; dim0 < values.n1(); ++dim0) { - for (int64 dim1 = 0; dim1 < values.n2(); ++dim1) { - for (int64 dim2 = 0; dim2 < values.n3(); ++dim2) { - Set({dim0, dim1, dim2}, values(dim0, dim1, dim2)); - } - } - } + PopulateFromArrayWithLayout(values, layout); } template void Literal::PopulateR3FromArray3D(const Array3D& values) { - PopulateR3FromArray3DWithLayout(values, LayoutUtil::GetDefaultLayoutForR3()); + PopulateFromArray(values); } template void Literal::PopulateR4FromArray4DWithLayout(const Array4D& values, const Layout& layout) { - *mutable_shape() = ShapeUtil::MakeShapeWithLayout( - primitive_util::NativeToPrimitiveType(), - {values.planes(), values.depth(), values.height(), values.width()}, - AsInt64Slice(layout.minor_to_major())); - - CHECK_EQ(values.n1(), shape().dimensions(0)); - CHECK_EQ(values.n2(), shape().dimensions(1)); - CHECK_EQ(values.n3(), shape().dimensions(2)); - CHECK_EQ(values.n4(), shape().dimensions(3)); - Reserve(values.n1() * values.n2() * values.n3() * values.n4()); - for (int64 dim0 = 0; dim0 < values.n1(); ++dim0) { - for (int64 dim1 = 0; dim1 < values.n2(); ++dim1) { - for (int64 dim2 = 0; dim2 < values.n3(); ++dim2) { - for (int64 dim3 = 0; dim3 < values.n4(); ++dim3) { - Set({dim0, dim1, dim2, dim3}, values(dim0, dim1, dim2, dim3)); - } - } - } - } + PopulateFromArrayWithLayout(values, layout); } template void Literal::PopulateR4FromArray4D(const Array4D& values) { - PopulateR4FromArray4DWithLayout(values, LayoutUtil::GetDefaultLayoutForR4()); + PopulateFromArray(values); } template diff --git a/tensorflow/compiler/xla/literal_util_test.cc b/tensorflow/compiler/xla/literal_util_test.cc index e7dedd08218d8a17c5e332e5cda7bedcc26f6703..a9af4849e2124fd47ae42cc06ac8cc5ca5a22cb7 100644 --- a/tensorflow/compiler/xla/literal_util_test.cc +++ b/tensorflow/compiler/xla/literal_util_test.cc @@ -107,6 +107,9 @@ TEST_F(LiteralUtilTest, LiteralScalarToString) { auto f16_lit = Literal::CreateR0(static_cast(0.5f)); ASSERT_EQ("0.5", f16_lit->ToString()); + + auto c64_lit = Literal::CreateR0({3.14f, 2.78f}); + ASSERT_EQ("(3.14, 2.78)", c64_lit->ToString()); } TEST_F(LiteralUtilTest, LiteralVectorToString) { @@ -331,6 +334,19 @@ TEST_F(LiteralUtilTest, TupleEquality) { EXPECT_NE(*tuple1, *different_tuple); } +TEST_F(LiteralUtilTest, C64Equality) { + // Test equality with tuples. + auto vector = Literal::CreateR1({{1.0, 2.0}, {3.0, 4.0}}); + + // Tuple with the same elements. One element is shared with the original + // tuple, the other is a clone of the element in the original tuple. + auto vector_clone = Literal::CreateR1({{1.0, 2.0}, {3.0, 4.0}}); + EXPECT_EQ(*vector, *vector_clone); + + auto vector_reversed = Literal::CreateR1({{3.0, 4.0}, {1.0, 2.0}}); + EXPECT_NE(*vector, *vector_reversed); +} + TEST_F(LiteralUtilTest, IsAllTuple) { auto element1 = Literal::CreateR0(0.0); auto element2 = Literal::CreateR2({{0.0, 0.0}, {0.0, 0.0}}); @@ -381,6 +397,9 @@ TEST_F(LiteralUtilTest, IsAll) { EXPECT_FALSE(Literal::CreateR2({{h8}, {h9}})->IsAll(8)); EXPECT_FALSE(Literal::CreateR2({{h9}, {h8}})->IsAll(8)); + complex64 c8_9 = {8, 9}; + EXPECT_FALSE(Literal::CreateR2({{c8_9}, {c8_9}})->IsAll(8)); + auto uint64_max = std::numeric_limits::max(); EXPECT_FALSE(Literal::CreateR2( {{uint64_max, uint64_max}, {uint64_max, uint64_max}}) @@ -411,6 +430,25 @@ TEST_F(LiteralUtilTest, IsAllFloat) { Literal::CreateR2({{0, 0, 0}, {0, .1, 0}})->IsAllFloat(0)); } +TEST_F(LiteralUtilTest, IsAllComplex) { + // IsAllComplex always returns false when the literal is not complex. + EXPECT_FALSE(Literal::CreateR0(false)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + EXPECT_FALSE(Literal::CreateR0(0)->IsAllComplex(0)); + + complex64 c8_9 = {8, 9}; + complex64 c7_9 = {7, 9}; + EXPECT_TRUE(Literal::CreateR2({{c8_9}, {c8_9}}) + ->IsAllComplex({8.0f, 9.0f})); + EXPECT_FALSE(Literal::CreateR2({{c7_9}, {c8_9}}) + ->IsAllComplex({8.0f, 9.0f})); + EXPECT_FALSE(Literal::CreateR2({{c8_9}, {c7_9}}) + ->IsAllComplex({8.0f, 9.0f})); +} + TEST_F(LiteralUtilTest, IsZero) { auto scalar_zero = Literal::CreateR0(0.0f); auto scalar_one = Literal::CreateR0(1.0f); @@ -422,12 +460,17 @@ TEST_F(LiteralUtilTest, IsZero) { EXPECT_TRUE(array->IsZero({0, 2})); EXPECT_TRUE(array->IsZero({1, 1})); EXPECT_FALSE(array->IsZero({1, 2})); + + auto complex_zero = Literal::CreateR0(0.0f); + auto complex_nonzero = Literal::CreateR0(0.5f); + EXPECT_TRUE(complex_zero->IsZero({})); + EXPECT_FALSE(complex_nonzero->IsZero({})); } template class LiteralUtilTestTemplated : public ::testing::Test {}; -using TestedTypes = ::testing::Types; +using TestedTypes = ::testing::Types; TYPED_TEST_CASE(LiteralUtilTestTemplated, TestedTypes); TYPED_TEST(LiteralUtilTestTemplated, Relayout2x2) { @@ -626,13 +669,28 @@ TEST_F(LiteralUtilTest, PopulateR1S64) { EXPECT_EQ(output, *expected); } -TEST_F(LiteralUtilTest, PopulateR2U64) { +TEST_F(LiteralUtilTest, PopulateR1U64) { Literal output; output.PopulateR1({{77, 88}}); auto expected = Literal::CreateR1({{77, 88}}); EXPECT_EQ(output, *expected); } +TEST_F(LiteralUtilTest, PopulateR1C64) { + Literal output; + output.PopulateR1({{77, 88}}); + auto expected = Literal::CreateR1({{77, 88}}); + EXPECT_EQ(output, *expected); +} + +TEST_F(LiteralUtilTest, PopulateR2C64) { + Literal output; + output.PopulateR2({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}}); + auto expected = + Literal::CreateR2({{{7, 8}, {9, 10}}, {{1, 2}, {3, 4}}}); + EXPECT_EQ(output, *expected); +} + TEST_F(LiteralUtilTest, PopulateWithValueR0F32) { Literal output; output.PopulateWithValue(2.5f, {}); @@ -654,6 +712,14 @@ TEST_F(LiteralUtilTest, PopulateWithValueR2U64) { EXPECT_EQ(output, *expected); } +TEST_F(LiteralUtilTest, PopulateWithValueR2C64) { + Literal output; + output.PopulateWithValue({4, 2}, {2, 2}); + auto expected = + Literal::CreateR2({{{4, 2}, {4, 2}}, {{4, 2}, {4, 2}}}); + EXPECT_EQ(output, *expected); +} + TEST_F(LiteralUtilTest, PopulateWithValueR0F16) { Literal output; half h(0.25f); @@ -919,6 +985,11 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) { {{0.0, 19.0, 0.0, 21.0}, {22.0, 0.0, 24.0, 0.0}}, {{26.0, 0.0, 28.0, 0.0}, {0.0, 31.0, 0.0, 33.0}}, }}, layout_r4_dim0major_); + auto c64 = Literal::CreateR4WithLayout({{ + {{10.0f, 0.0f, 12.0f, 0.0f}, {0.0f, 15.0f, 0.0f, 17.0f}}, + {{0.0f, 19.0f, 0.0f, 21.0f}, {22.0f, 0.0f, 24.0f, 0.0f}}, + {{26.0f, 0.0f, 28.0f, 0.0f}, {0.0f, 31.0f, 0.0f, 33.0f}}, + }}, layout_r4_dim0major_); // clang-format on std::unique_ptr conv; @@ -961,12 +1032,22 @@ TEST_F(LiteralUtilTest, ConvertIfTypesMatch) { conv = u32->Convert(F16).ConsumeValueOrDie(); EXPECT_EQ(*conv, *f16); + conv = s32->Convert(C64).ConsumeValueOrDie(); + EXPECT_EQ(*conv, *c64); + + conv = f16->Convert(C64).ConsumeValueOrDie(); + EXPECT_EQ(*conv, *c64); + EXPECT_EQ(s32->Convert(TUPLE).status().code(), tensorflow::error::INVALID_ARGUMENT); EXPECT_EQ(s32->Convert(S16).status().code(), tensorflow::error::INVALID_ARGUMENT); EXPECT_EQ(s32->Convert(U16).status().code(), tensorflow::error::INVALID_ARGUMENT); + EXPECT_EQ(c64->Convert(F32).status().code(), + tensorflow::error::INVALID_ARGUMENT); + EXPECT_EQ(c64->Convert(S32).status().code(), + tensorflow::error::INVALID_ARGUMENT); } TEST_F(LiteralUtilTest, CopyFromProto_Bool) { diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc index e4e37177a2d74e6da20300f1439942a146ad8d49..2113b5e06f3eb0169be50c0ee731a903c0eece9d 100644 --- a/tensorflow/compiler/xla/primitive_util.cc +++ b/tensorflow/compiler/xla/primitive_util.cc @@ -83,10 +83,17 @@ PrimitiveType NativeToPrimitiveType() { return F16; } +template <> +PrimitiveType NativeToPrimitiveType() { + return C64; +} + bool IsFloatingPointType(PrimitiveType type) { return type == F16 || type == F32 || type == F64; } +bool IsComplexType(PrimitiveType type) { return type == C64; } + bool IsSignedIntegralType(PrimitiveType type) { return type == S8 || type == S16 || type == S32 || type == S64; } @@ -121,6 +128,7 @@ int BitWidth(PrimitiveType type) { case U64: case S64: case F64: + case C64: return 64; case TUPLE: @@ -134,5 +142,15 @@ int BitWidth(PrimitiveType type) { } } +PrimitiveType ComplexComponentType(PrimitiveType complex_type) { + switch (complex_type) { + case C64: + return F32; + default: + LOG(FATAL) << "Primitive type is not complex: " + << PrimitiveType_Name(complex_type); + } +} + } // namespace primitive_util } // namespace xla diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h index 162a11c7d2966346979b98c804917203f82c806c..a49c8b86fcfe156ea3733ce05c0fb7337cf60dce 100644 --- a/tensorflow/compiler/xla/primitive_util.h +++ b/tensorflow/compiler/xla/primitive_util.h @@ -78,8 +78,14 @@ PrimitiveType NativeToPrimitiveType(); template <> PrimitiveType NativeToPrimitiveType(); +// Complex +template <> +PrimitiveType NativeToPrimitiveType(); + bool IsFloatingPointType(PrimitiveType type); +bool IsComplexType(PrimitiveType type); + bool IsSignedIntegralType(PrimitiveType type); bool IsUnsignedIntegralType(PrimitiveType type); @@ -89,6 +95,10 @@ bool IsIntegralType(PrimitiveType type); // Returns the number of bits in the representation for a given type. int BitWidth(PrimitiveType type); +// Returns the real, imag component type underlying the given complex type. +// LOG(FATAL)'s if complex_type is not complex. +PrimitiveType ComplexComponentType(PrimitiveType complex_type); + // Returns the native type (eg, float) corresponding to the given template // parameter XLA primitive type (eg, F32). template @@ -157,6 +167,11 @@ struct PrimitiveTypeToNative { using type = half; }; +// Complex +template <> +struct PrimitiveTypeToNative { + using type = complex64; +}; } // namespace primitive_util } // namespace xla diff --git a/tensorflow/compiler/xla/protobuf_util.cc b/tensorflow/compiler/xla/protobuf_util.cc index c032cb8dc5adcbef9ffa64aa1e05bb5ccb49fc6a..787725e884c810fd724ab88ad7d4beaf3e0a6cc7 100644 --- a/tensorflow/compiler/xla/protobuf_util.cc +++ b/tensorflow/compiler/xla/protobuf_util.cc @@ -37,20 +37,6 @@ bool ProtobufEquals(const tensorflow::protobuf::Message& m1, return (serialized1 == serialized2); } -StatusOr ToJson(const tensorflow::protobuf::Message& message) { - string json_output; - tensorflow::protobuf::util::JsonPrintOptions json_options; - json_options.add_whitespace = true; - json_options.always_print_primitive_fields = true; - auto status = tensorflow::protobuf::util::MessageToJsonString( - message, &json_output, json_options); - if (!status.ok()) { - return InternalError("MessageToJsonString failed: %s", - status.error_message().data()); - } - return json_output; -} - namespace { string SanitizeFilename(const string& file_name) { @@ -65,17 +51,6 @@ string SanitizeFilename(const string& file_name) { } // namespace -Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message, - const string& directory, const string& file_name) { - TF_ASSIGN_OR_RETURN(const string json_output, ToJson(message)); - - tensorflow::Env* env = tensorflow::Env::Default(); - TF_RETURN_IF_ERROR(env->RecursivelyCreateDir(directory)); - string safe_file_name = SanitizeFileName(file_name) + ".json"; - const string path = tensorflow::io::JoinPath(directory, safe_file_name); - return tensorflow::WriteStringToFile(env, path, json_output); -} - Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message, const string& directory, const string& file_name) { tensorflow::Env* env = tensorflow::Env::Default(); diff --git a/tensorflow/compiler/xla/protobuf_util.h b/tensorflow/compiler/xla/protobuf_util.h index 7accb22e0c7720d5af896f8ca833ee26175fb89f..3667621367c7639c40ff17aee7b77305d4d34e33 100644 --- a/tensorflow/compiler/xla/protobuf_util.h +++ b/tensorflow/compiler/xla/protobuf_util.h @@ -32,17 +32,12 @@ namespace protobuf_util { extern bool ProtobufEquals(const tensorflow::protobuf::Message& m1, const tensorflow::protobuf::Message& m2); -// Returns 'message' as a JSON string. -StatusOr ToJson(const tensorflow::protobuf::Message& message); - -// Writes the given message in binary proto or JSON format to the path formed by -// joining 'directory/file_name.pb' (or file_name.json). The 'directory' is -// recursively created if it doesn't already exist, and the 'file_name' is -// sanitized by replacing illegal characters with underscore '_'. +// Writes the given message in binary proto to the path formed by joining +// 'directory/file_name.pb'. The 'directory' is recursively created if it +// doesn't already exist, and the 'file_name' is sanitized by replacing +// illegal characters with underscore '_'. Status DumpProtoToDirectory(const tensorflow::protobuf::Message& message, const string& directory, const string& file_name); -Status DumpJsonToDirectory(const tensorflow::protobuf::Message& message, - const string& directory, const string& file_name); } // namespace protobuf_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 9756dc781722b34a4b265d2d6e7afb41207396e0..a15f3f654b14a715a2fbc71cdd38d46ac0268c02 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -115,7 +115,7 @@ tf_cc_test( "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/client:computation_builder", - "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:hlo_verified_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", # fixdeps: keep "//tensorflow/core:lib", @@ -131,6 +131,7 @@ cc_library( "hlo_instruction.cc", "hlo_module.cc", "hlo_opcode.cc", + "hlo_sharding.cc", ], hdrs = [ "dfs_hlo_visitor.h", @@ -139,6 +140,7 @@ cc_library( "hlo_instruction.h", "hlo_module.h", "hlo_opcode.h", + "hlo_sharding.h", ], deps = [ ":hlo_module_config", @@ -146,6 +148,7 @@ cc_library( ":hlo_reachability", ":name_uniquer", ":versioned_computation_handle", + "//tensorflow/compiler/xla:array", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:shape_tree", @@ -236,6 +239,22 @@ tf_cc_test( ], ) +tf_cc_test( + name = "hlo_sharding_test", + srcs = ["hlo_sharding_test.cc"], + deps = [ + ":hlo", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:protobuf_util", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla:test_helpers", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + ], +) + cc_library( name = "call_graph", srcs = ["call_graph.cc"], @@ -431,7 +450,6 @@ cc_library( ":hlo_evaluator", ":hlo_execution_profile", ":hlo_module_config", - ":hlo_verifier", ":platform_util", ":session_proto", ":transfer_manager", @@ -580,12 +598,14 @@ cc_library( ":shaped_buffer", ":versioned_computation_handle", "//tensorflow/compiler/xla:executable_run_options", + "//tensorflow/compiler/xla:status", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/legacy_flags:debug_options_flags", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/stream_executor", ], @@ -1071,6 +1091,33 @@ tf_cc_test( ], ) +cc_library( + name = "defuser", + srcs = ["defuser.cc"], + hdrs = ["defuser.h"], + deps = [ + ":call_graph", + ":hlo", + ":hlo_pass", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "defuser_test", + srcs = ["defuser_test.cc"], + deps = [ + ":defuser", + ":hlo_matchers", + "//tensorflow/compiler/xla:literal_util", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla/tests:hlo_verified_test_base", + ], +) + cc_library( name = "tuple_simplifier", srcs = ["tuple_simplifier.cc"], @@ -2063,6 +2110,29 @@ tf_cc_test( ], ) +cc_library( + name = "hlo_runner", + srcs = ["hlo_runner.cc"], + hdrs = ["hlo_runner.h"], + deps = [ + ":executable", + ":hlo", + ":transfer_manager", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:backend", + "//tensorflow/compiler/xla/service:compiler", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:lib", + "//tensorflow/core:stream_executor_no_cuda", + "//third_party/eigen3", + ], +) + # ----------------------------------------------------------------------------- filegroup( diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 39e8430ed335806a8b71f391ecfb30e2e3716633..ee5cf8a10074d72d81374cf9dcb2cb2164f0d9db 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -123,71 +123,54 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleAdd(HloInstruction* add) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleBroadcast(HloInstruction* broadcast) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; Status HandleCopy(HloInstruction* copy) override; Status HandleConvert(HloInstruction* convert) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleReal(HloInstruction* real) override; + Status HandleImag(HloInstruction* imag) override; - Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleConvolution(HloInstruction* convolution) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleDivide(HloInstruction* divide) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleDot(HloInstruction* dot) override; - Status HandleLog(HloInstruction* log, HloInstruction* operand) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; - Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleLog(HloInstruction* log) override; + + Status HandleMultiply(HloInstruction* multiply) override; Status HandlePad(HloInstruction* pad) override; - Status HandlePower(HloInstruction* power, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandlePower(HloInstruction* power) override; Status HandleReshape(HloInstruction* reshape) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; - - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override; - - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override; - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override; - Status HandleDynamicSlice(HloInstruction* slice, HloInstruction* operand, - HloInstruction* start_indices) override; - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override; + Status HandleReduce(HloInstruction* reduce) override; + + Status HandleReduceWindow(HloInstruction* reduce_window) override; + + Status HandleReverse(HloInstruction* reverse) override; + Status HandleSlice(HloInstruction* slice) override; + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override; + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override; Status HandleTranspose(HloInstruction* transpose) override; - Status HandleSubtract(HloInstruction* sub, HloInstruction* lhs, - HloInstruction* rhs) override; + Status HandleSubtract(HloInstruction* sub) override; Status HandleMaximum(HloInstruction* maximum) override; Status HandleMinimum(HloInstruction* minimum) override; @@ -201,17 +184,18 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { static bool Run( HloComputation* computation, bool is_layout_sensitive, AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification); + bool enable_dot_simplification, bool enable_conv_simplification); private: explicit AlgebraicSimplifierVisitor( HloComputation* computation, bool is_layout_sensitive, AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification) + bool enable_dot_simplification, bool enable_conv_simplification) : computation_(computation), is_layout_sensitive_(is_layout_sensitive), valid_bitcast_callback_(std::move(valid_bitcast_callback)), - enable_dot_simplification_(enable_dot_simplification) {} + enable_dot_simplification_(enable_dot_simplification), + enable_conv_simplification_(enable_conv_simplification) {} // Convenience method for replacing an instruction with a bitcast. void ReplaceWithBitcast(HloInstruction* instruction); @@ -287,15 +271,18 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault { // Disable dot simplication on platforms where it causes a slowdown. bool enable_dot_simplification_; + + // Disable convolution simplication on platforms where it causes a slowdown. + bool enable_conv_simplification_; }; bool AlgebraicSimplifierVisitor::Run( HloComputation* computation, bool is_layout_sensitive, AlgebraicSimplifier::ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification) { - AlgebraicSimplifierVisitor visitor(computation, is_layout_sensitive, - std::move(valid_bitcast_callback), - enable_dot_simplification); + bool enable_dot_simplification, bool enable_conv_simplification) { + AlgebraicSimplifierVisitor visitor( + computation, is_layout_sensitive, std::move(valid_bitcast_callback), + enable_dot_simplification, enable_conv_simplification); TF_CHECK_OK(computation->Accept(&visitor)); return visitor.changed_; } @@ -332,9 +319,9 @@ bool AlgebraicSimplifierVisitor::ReplaceInstructionIfSameShape( return true; } -Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add) { + auto lhs = add->mutable_operand(0); + auto rhs = add->mutable_operand(1); // A + 0 => A VLOG(10) << "trying transform [A + 0 => A]: " << add->ToString(); if (IsAll(rhs, 0) && ReplaceInstructionIfSameShape(add, lhs)) { @@ -377,8 +364,9 @@ Status AlgebraicSimplifierVisitor::HandleCopy(HloInstruction* copy) { } Status AlgebraicSimplifierVisitor::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { + HloInstruction* concatenate) { + tensorflow::gtl::ArraySlice operands( + concatenate->operands()); if (operands.size() == 1) { // Unary concatenates are useless. ReplaceInstructionIfSameShape(concatenate, operands[0]); @@ -459,20 +447,19 @@ static HloInstruction* BuildTupleConstant(HloComputation* computation, } } -Status AlgebraicSimplifierVisitor::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status AlgebraicSimplifierVisitor::HandleConstant(HloInstruction* constant) { // Tuple constants aren't directly supported by any backend. Expand them into // explicit Tuple instructions. if (ShapeUtil::IsTuple(constant->shape())) { - return ReplaceInstruction(constant, - BuildTupleConstant(computation_, literal)); + return ReplaceInstruction( + constant, BuildTupleConstant(computation_, constant->literal())); } return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleSubtract(HloInstruction* sub, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleSubtract(HloInstruction* sub) { + auto lhs = sub->mutable_operand(0); + auto rhs = sub->mutable_operand(1); // A - 0 => A VLOG(10) << "trying transform [A - 0 => A]: " << sub->ToString(); if (IsAll(rhs, 0) && ReplaceInstructionIfSameShape(sub, lhs)) { @@ -482,9 +469,9 @@ Status AlgebraicSimplifierVisitor::HandleSubtract(HloInstruction* sub, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) { + auto lhs = divide->mutable_operand(0); + auto rhs = divide->mutable_operand(1); // A/1 => A VLOG(10) << "trying transform [A/1 => A]: " << divide->ToString(); if (IsAll(rhs, 1) && ReplaceInstructionIfSameShape(divide, lhs)) { @@ -519,11 +506,16 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide, // A/pow(B,C) => A*pow(B,-C) if (rhs->opcode() == HloOpcode::kPower) { VLOG(10) << "transform [A/pow(B,C) => A*pow(B,-C)]: " << divide->ToString(); + // The output shape of the created negate operator should be the same as the + // input. + const Shape& negate_shape = rhs->operand(1)->shape(); HloInstruction* negate = computation_->AddInstruction(HloInstruction::CreateUnary( - divide->shape(), HloOpcode::kNegate, rhs->mutable_operand(1))); + negate_shape, HloOpcode::kNegate, rhs->mutable_operand(1))); + // And the power operator should retain the output shape of the old one. + const Shape& new_power_shape = rhs->shape(); HloInstruction* new_power = computation_->AddInstruction( - HloInstruction::CreateBinary(divide->shape(), HloOpcode::kPower, + HloInstruction::CreateBinary(new_power_shape, HloOpcode::kPower, rhs->mutable_operand(0), negate)); return ReplaceWithNewInstruction( divide, HloInstruction::CreateBinary( @@ -586,9 +578,9 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot) { + auto lhs = dot->mutable_operand(0); + auto rhs = dot->mutable_operand(1); if (!enable_dot_simplification_) { return Status::OK(); } @@ -717,9 +709,9 @@ Status AlgebraicSimplifierVisitor::HandleDot(HloInstruction* dot, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply) { + auto lhs = multiply->mutable_operand(0); + auto rhs = multiply->mutable_operand(1); // A*1 => A VLOG(10) << "trying transform [A*1 => A]: " << multiply->ToString(); if (IsAll(rhs, 1) && ReplaceInstructionIfSameShape(multiply, lhs)) { @@ -743,10 +735,10 @@ Status AlgebraicSimplifierVisitor::HandleMultiply(HloInstruction* multiply, return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleLog(HloInstruction* log, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleLog(HloInstruction* log) { // ln(exp(A)) => A VLOG(10) << "trying transform [ln(exp(A)) => A]: " << log->ToString(); + auto operand = log->mutable_operand(0); if (operand->opcode() == HloOpcode::kExp && ReplaceInstructionIfSameShape(log, operand->mutable_operand(0))) { return Status::OK(); @@ -766,7 +758,8 @@ Status AlgebraicSimplifierVisitor::HandleLog(HloInstruction* log, } Status AlgebraicSimplifierVisitor::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { + HloInstruction* get_tuple_element) { + auto operand = get_tuple_element->mutable_operand(0); if (operand->opcode() == HloOpcode::kTuple) { // get_tuple_element(make_tuple({A_0, A_1, ..., A_n}), i) => A_i VLOG(10) << "trying transform " @@ -958,6 +951,24 @@ Status AlgebraicSimplifierVisitor::HandleConvert(HloInstruction* convert) { return Status::OK(); } +// Real(Complex(r, i)) -> r +Status AlgebraicSimplifierVisitor::HandleReal(HloInstruction* real) { + auto operand = real->mutable_operand(0); + if (operand->opcode() == HloOpcode::kComplex) { + return ReplaceInstruction(real, operand->mutable_operand(0)); + } + return Status::OK(); +} + +// Imag(Complex(r, i)) -> i +Status AlgebraicSimplifierVisitor::HandleImag(HloInstruction* imag) { + auto operand = imag->mutable_operand(0); + if (operand->opcode() == HloOpcode::kComplex) { + return ReplaceInstruction(imag, operand->mutable_operand(1)); + } + return Status::OK(); +} + Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) { // Eliminate nop pads (padding all zero), and replace a pad with negative // padding with a pad with non-negative padding followed by a slice. @@ -1048,10 +1059,10 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) { return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandlePower(HloInstruction* power, - HloInstruction* lhs, - HloInstruction* rhs) { +Status AlgebraicSimplifierVisitor::HandlePower(HloInstruction* power) { VLOG(10) << "trying transform [pow(A, 0) => 1]: " << power->ToString(); + auto lhs = power->mutable_operand(0); + auto rhs = power->mutable_operand(1); if (IsAll(rhs, 0)) { auto one = HloInstruction::CreateConstant( Literal::One(power->shape().element_type()).CloneToUnique()); @@ -1235,8 +1246,7 @@ Status AlgebraicSimplifierVisitor::HandleReshape(HloInstruction* reshape) { return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse) { // When all the dimensions to reverse are trivial (i.e. the bound is 1), // there is nothing to be done. auto dim_is_one = [&](int64 i) -> bool { @@ -1244,23 +1254,23 @@ Status AlgebraicSimplifierVisitor::HandleReverse(HloInstruction* reverse, }; if (std::all_of(reverse->dimensions().begin(), reverse->dimensions().end(), dim_is_one)) { - return ReplaceInstruction(reverse, operand); + return ReplaceInstruction(reverse, reverse->mutable_operand(0)); } return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleSlice(HloInstruction* slice, - HloInstruction* operand) { +Status AlgebraicSimplifierVisitor::HandleSlice(HloInstruction* slice) { // Delete no-op slices, i.e. where shape = operand shape. - if (ReplaceInstructionIfSameShape(slice, operand)) { + if (ReplaceInstructionIfSameShape(slice, slice->mutable_operand(0))) { return Status::OK(); } return Status::OK(); } Status AlgebraicSimplifierVisitor::HandleDynamicSlice( - HloInstruction* dynamic_slice, HloInstruction* operand, - HloInstruction* start_indices) { + HloInstruction* dynamic_slice) { + auto operand = dynamic_slice->mutable_operand(0); + auto start_indices = dynamic_slice->operand(1); if (ShapeUtil::IsScalar(dynamic_slice->shape())) { return ReplaceInstruction(dynamic_slice, operand); } @@ -1273,8 +1283,9 @@ Status AlgebraicSimplifierVisitor::HandleDynamicSlice( } Status AlgebraicSimplifierVisitor::HandleDynamicUpdateSlice( - HloInstruction* dynamic_update_slice, HloInstruction* operand, - HloInstruction* update, HloInstruction* start_indices) { + HloInstruction* dynamic_update_slice) { + auto update = dynamic_update_slice->mutable_operand(1); + auto start_indices = dynamic_update_slice->operand(2); // DynamicUpdateSlice on a scalar just passes through the update argument. if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) { return ReplaceInstruction(dynamic_update_slice, update); @@ -1293,9 +1304,11 @@ Status AlgebraicSimplifierVisitor::HandleDynamicUpdateSlice( return Status::OK(); } -Status AlgebraicSimplifierVisitor::HandleReduce( - HloInstruction* reduce, HloInstruction* arg, HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, HloComputation* function) { +Status AlgebraicSimplifierVisitor::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->mutable_operand(0); + auto init_value = reduce->mutable_operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); if (ShapeUtil::HasZeroElements(arg->shape()) || ShapeUtil::HasZeroElements(reduce->shape())) { return ReplaceWithNewInstruction( @@ -1373,8 +1386,10 @@ Status AlgebraicSimplifierVisitor::HandleReduce( } Status AlgebraicSimplifierVisitor::HandleReduceWindow( - HloInstruction* reduce_window, HloInstruction* operand, - const Window& window, HloComputation* function) { + HloInstruction* reduce_window) { + auto operand = reduce_window->mutable_operand(0); + const Window& window = reduce_window->window(); + auto function = reduce_window->to_apply(); VLOG(10) << "Considering folding Pad: " << operand->ToString() << "\ninto reduce-window: " << reduce_window->ToString(); @@ -1457,8 +1472,13 @@ Status AlgebraicSimplifierVisitor::HandleTranspose(HloInstruction* transpose) { } Status AlgebraicSimplifierVisitor::HandleConvolution( - HloInstruction* convolution, HloInstruction* lhs, HloInstruction* rhs, - const Window& window) { + HloInstruction* convolution) { + auto lhs = convolution->mutable_operand(0); + auto rhs = convolution->mutable_operand(1); + const auto& window = convolution->window(); + if (!enable_conv_simplification_) { + return Status::OK(); + } // HandleConvolution tries to replace a convolution with a DOT instruction. // // Only add when bitcasts can be used: @@ -1929,7 +1949,7 @@ Status AlgebraicSimplifierVisitor::HandleWhile(HloInstruction* while_op) { return Status::OK(); } - // Remove while loops with static trip count of 1. + // Remove while loops with static trip count of 0. optional trip_count = GetLoopTripCount(while_op); if (trip_count && *trip_count == 0) { // The loop never executes, so the value of the loop is the value of its @@ -1944,8 +1964,10 @@ Status AlgebraicSimplifierVisitor::HandleWhile(HloInstruction* while_op) { changed_ = true; return Status::OK(); } + + // Transform while loops with static trip count of 1 into a call op, then + // inline the call. if (trip_count && *trip_count == 1) { - // Transform the while loop into a call op, then inline the call. auto computation = while_op->parent(); auto call_op = computation->AddInstruction(HloInstruction::CreateCall( while_op->shape(), while_op->operands(), while_op->while_body())); @@ -1962,9 +1984,9 @@ StatusOr AlgebraicSimplifier::Run(HloModule* module) { "AlgebraicSimplifier::Run(), before:\n" + module->ToString()); bool changed = false; for (auto* comp : module->MakeNonfusionComputations()) { - if (AlgebraicSimplifierVisitor::Run(comp, is_layout_sensitive_, - valid_bitcast_callback_, - enable_dot_simplification_)) { + if (AlgebraicSimplifierVisitor::Run( + comp, is_layout_sensitive_, valid_bitcast_callback_, + enable_dot_simplification_, enable_conv_simplification_)) { changed = true; } } diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h index 4295a3227a837ffc8483b3be59994c9e6ac96aec..a9f476178c7af74c275a10de7727ea64e17d590f 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.h +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h @@ -40,11 +40,13 @@ class AlgebraicSimplifier : public HloPassInterface { // bitcasts. AlgebraicSimplifier(bool is_layout_sensitive, ValidBitcastCallback valid_bitcast_callback, - bool enable_dot_simplification = true) + bool enable_dot_simplification = true, + bool enable_conv_simplification = true) : is_layout_sensitive_(is_layout_sensitive), valid_bitcast_callback_(std::move(valid_bitcast_callback)), - enable_dot_simplification_(enable_dot_simplification) {} - ~AlgebraicSimplifier() override {} + enable_dot_simplification_(enable_dot_simplification), + enable_conv_simplification_(enable_conv_simplification) {} + ~AlgebraicSimplifier() override = default; tensorflow::StringPiece name() const override { return "algsimp"; } // Run algebraic simplification on the given computation. Returns whether the @@ -57,6 +59,9 @@ class AlgebraicSimplifier : public HloPassInterface { // Enable dot simplication on platforms where it is profitable. bool enable_dot_simplification_; + + // Enable convolution simplication on platforms where it is profitable. + bool enable_conv_simplification_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index af502206e2ba85d89e208e0b8697273d2bf9b7ab..87d4fc9663daf3cc2806dfa6550812dd9b08b36c 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -353,6 +353,42 @@ TEST_F(AlgebraicSimplifierTest, DivOfPower) { op::Multiply(param0, op::Power(param1, op::Negate(param2)))); } +// Test that broadcasting is done on the right step when simplifying A/pow(B,C) +// to A*pow(B,-C). +TEST_F(AlgebraicSimplifierTest, DivOfBroadcastingPower) { + Shape r0f32 = ShapeUtil::MakeShape(F32, {}); + Shape r1f32 = ShapeUtil::MakeShape(F32, {7}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r1f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r1f32, "param1")); + HloInstruction* param2 = builder.AddInstruction( + HloInstruction::CreateParameter(2, r0f32, "param2")); + HloInstruction* power = builder.AddInstruction( + HloInstruction::CreateBinary(r1f32, HloOpcode::kPower, param1, param2)); + builder.AddInstruction( + HloInstruction::CreateBinary(r1f32, HloOpcode::kDivide, param0, power)); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + + EXPECT_THAT(computation->root_instruction(), + op::Divide(param0, op::Power(param1, param2))); + + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + + ASSERT_THAT(computation->root_instruction(), + op::Multiply(param0, op::Power(param1, op::Negate(param2)))); + + const HloInstruction* negate = + computation->root_instruction()->operand(1)->operand(1); + const Shape& negate_shape = negate->shape(); + EXPECT_EQ(0, negate_shape.dimensions_size()); +} + // Test that A/1 is simplified to A for a scalar. TEST_F(AlgebraicSimplifierTest, DivOneScalar) { Shape r0f32 = ShapeUtil::MakeShape(F32, {}); @@ -397,6 +433,56 @@ TEST_F(AlgebraicSimplifierTest, DivOneArray) { EXPECT_EQ(root, param0); } +// Test that real(complex(r,i)) is simplified to r. +TEST_F(AlgebraicSimplifierTest, RealOfComplex) { + Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r2f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r2f32, "param1")); + HloInstruction* cplx = builder.AddInstruction( + HloInstruction::CreateBinary(ShapeUtil::ChangeElementType(r2f32, C64), + HloOpcode::kComplex, param0, param1)); + HloInstruction* real = builder.AddInstruction( + HloInstruction::CreateUnary(r2f32, HloOpcode::kReal, cplx)); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + HloInstruction* root = computation->root_instruction(); + EXPECT_EQ(root, real); + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + root = computation->root_instruction(); + EXPECT_EQ(root, param0); +} + +// Test that imag(complex(r,i)) is simplified to i. +TEST_F(AlgebraicSimplifierTest, ImagOfComplex) { + Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2}); + HloComputation::Builder builder(TestName()); + HloInstruction* param0 = builder.AddInstruction( + HloInstruction::CreateParameter(0, r2f32, "param0")); + HloInstruction* param1 = builder.AddInstruction( + HloInstruction::CreateParameter(1, r2f32, "param1")); + HloInstruction* cplx = builder.AddInstruction( + HloInstruction::CreateBinary(ShapeUtil::ChangeElementType(r2f32, C64), + HloOpcode::kComplex, param0, param1)); + HloInstruction* imag = builder.AddInstruction( + HloInstruction::CreateUnary(r2f32, HloOpcode::kImag, cplx)); + + auto module = CreateNewModule(); + auto computation = module->AddEntryComputation(builder.Build()); + HloInstruction* root = computation->root_instruction(); + EXPECT_EQ(root, imag); + AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false, + non_bitcasting_callback()); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + root = computation->root_instruction(); + EXPECT_EQ(root, param1); +} + // Test that get_element(make_tuple({A,B}),1) is simplified to B TEST_F(AlgebraicSimplifierTest, SelectMakeTuple) { Shape r0f32 = ShapeUtil::MakeShape(F32, {}); @@ -2091,7 +2177,7 @@ TEST_F(AlgebraicSimplifierTest, IteratorInvalidation) { HloInstruction::CreateConstant(Literal::CreateR1({0.0f}))); HloInstruction* one = call_builder.AddInstruction( HloInstruction::CreateConstant(Literal::CreateR1({1.0f}))); - builder.AddInstruction( + call_builder.AddInstruction( HloInstruction::CreateCall(r1f32, {zero, one}, dot_computation.get())); auto module = CreateNewModule(); diff --git a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc index 427294dfc6fa4a27e28dc0fcb0f726601aa94468..abe881cd1a58a6173b9b93f10a7308d70106c889 100644 --- a/tensorflow/compiler/xla/service/batchnorm_rewriter.cc +++ b/tensorflow/compiler/xla/service/batchnorm_rewriter.cc @@ -83,11 +83,11 @@ class BatchNormRewriterVisitor : public DfsHloVisitorWithDefault { HloComputation* GetScalarBinaryComputation(PrimitiveType primitive_type, HloOpcode opcode) { - HloComputation::Builder b("scalar computation"); + HloComputation::Builder b("scalar_computation"); auto scalar_lhs = b.AddInstruction(HloInstruction::CreateParameter( - 0, ShapeUtil::MakeShape(F32, {}), "scalar lhs")); + 0, ShapeUtil::MakeShape(F32, {}), "scalar_lhs")); auto scalar_rhs = b.AddInstruction(HloInstruction::CreateParameter( - 1, ShapeUtil::MakeShape(F32, {}), "scalar rhs")); + 1, ShapeUtil::MakeShape(F32, {}), "scalar_rhs")); auto scalar_op = b.AddInstruction( HloInstruction::CreateBinary(ShapeUtil::MakeShape(primitive_type, {}), opcode, scalar_lhs, scalar_rhs)); diff --git a/tensorflow/compiler/xla/service/buffer_assignment_test.cc b/tensorflow/compiler/xla/service/buffer_assignment_test.cc index e3378a756b383a17a937f55afcd9ac08fe175fec..89410f42bd7b5fa8f9b380c868fcd4fedb54576c 100644 --- a/tensorflow/compiler/xla/service/buffer_assignment_test.cc +++ b/tensorflow/compiler/xla/service/buffer_assignment_test.cc @@ -1179,7 +1179,7 @@ TEST_F(BufferAssignmentTest, TupleCallAsOutput) { auto assignment = RunBufferAssignment(module.get()); EXPECT_EQ(3, assignment->Allocations().size()); - // Buffers for call are co-located with the sub-computation. + // Buffers for call are colocated with the sub-computation. EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{}), GetAllocation(*assignment, sub_tuple, /*index=*/{})); EXPECT_EQ(GetAllocation(*assignment, call, /*index=*/{0}), @@ -1238,7 +1238,7 @@ TEST_F(BufferAssignmentTest, TupleChainedCallAsOutput) { auto assignment = RunBufferAssignment(module.get()); - // Buffers for call are co-located with the sub-computations. + // Buffers for call are colocated with the sub-computations. EXPECT_EQ(GetAllocation(*assignment, a_call, /*index=*/{}), GetAllocation(*assignment, b_call, /*index=*/{})); EXPECT_EQ(GetAllocation(*assignment, b_call, /*index=*/{}), diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 8ab358fe17543735d87d457aa45f7b32f695c4b5..ef8eed3f88c3d557fcb4ec5b9e1988ce82b777e8 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -87,6 +87,7 @@ cc_library( ":ir_emitter", ":layout_assignment", ":parallel_cpu_executable", + ":parallel_task_assignment", ":simple_orc_jit", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:protobuf_util", @@ -145,16 +146,17 @@ cc_library( name = "simple_orc_jit", srcs = ["simple_orc_jit.cc"], hdrs = ["simple_orc_jit.h"], - linkopts = ["-ldl"], deps = [ ":compiler_functor", ":cpu_runtime", ":cpu_runtime_avx", ":cpu_runtime_neon", ":cpu_runtime_sse4_1", + ":custom_call_target_registry", ":disassembler", ":external_constant_pool", ":runtime_conv2d", + ":runtime_fork_join", ":runtime_matmul", ":runtime_single_threaded_conv2d", ":runtime_single_threaded_matmul", @@ -243,6 +245,7 @@ cc_library( ":dot_op_emitter", ":external_constant_pool", ":ir_emission_utils", + ":shape_partition", ":simple_orc_jit", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -505,9 +508,24 @@ cc_library( ], ) +cc_library( + name = "runtime_fork_join", + srcs = ["runtime_fork_join.cc"], + hdrs = ["runtime_fork_join.h"], + copts = runtime_copts(), + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/xla:executable_run_options", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//third_party/eigen3", + ], +) + tf_cc_test( name = "cpu_runtime_test", srcs = ["cpu_runtime_test.cc"], + tags = ["optonly"], deps = [ ":cpu_runtime", ":runtime_matmul", @@ -688,6 +706,7 @@ cc_library( ":shape_partition", "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service:hlo_cost_analysis", + "//tensorflow/compiler/xla/service:hlo_pass", ], ) @@ -700,6 +719,17 @@ cc_library( ], ) +cc_library( + name = "custom_call_target_registry", + srcs = [ + "custom_call_target_registry.cc", + ], + hdrs = [ + "custom_call_target_registry.h", + ], + visibility = ["//visibility:public"], +) + # ----------------------------------------------------------------------------- filegroup( diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 1437fb4cf93e3b0552fe85a584cb78a8d7e58dba..e141066b8fb48896e9f88e0a98f74aad08b63799 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -58,6 +58,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/cpu/layout_assignment.h" #include "tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h" +#include "tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/flatten_call_graph.h" @@ -221,14 +222,9 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault { } // Skip constants, there is nothing to profile. - Status HandleConstant(HloInstruction* /*constant*/, - const Literal& /*literal*/) override { - return Status::OK(); - } + Status HandleConstant(HloInstruction*) override { return Status::OK(); } // Skip parameters, they are a simple load. - Status HandleParameter(HloInstruction* /*parameter*/) override { - return Status::OK(); - } + Status HandleParameter(HloInstruction*) override { return Status::OK(); } // It is important to recurse for "while" or else we risk overly coarse // profiling information. Status HandleWhile(HloInstruction* xla_while) override { @@ -248,7 +244,7 @@ class CollectProfileCandidates : public DfsHloVisitorWithDefault { }; } // namespace -Status CpuCompiler::RunHloPasses(HloModule* module) { +Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) { // Optimization pipeline. HloPassPipeline pipeline("CPU"); pipeline.AddInvariantChecker(ShapeSizeBytesFunction()); @@ -281,6 +277,7 @@ Status CpuCompiler::RunHloPasses(HloModule* module) { [](const Shape&, const Shape&) { return false; }, /*enable_dot_simplification=*/false); pass.AddPass(); + pass.AddPass(); pass.AddPass(); pass.AddPass(); } @@ -316,6 +313,14 @@ Status CpuCompiler::RunHloPasses(HloModule* module) { if (options::CpuParallelBackendRequested(module->config())) { pipeline.AddPass(max_parallelism, ShapeSizeBytesFunction()); + } else if (!is_aot_compile) { + // Run ParallelTaskAssigner to assign parallel tasks to HLOs in module. + // Note this is not run for AOT because it would bring in thread pool + // and thread synchronization dependencies which would likely increase + // binary size (and most AOT applications are single-threaded). + // TODO(29630486) Support multi-threaded AOT. + pipeline.AddPass(max_parallelism, + ShapeSizeBytesFunction(), module); } // Copy insertion should be performed immediately before IR emission to avoid // inserting unnecessary copies (later pass adds an instruction which @@ -450,7 +455,7 @@ StatusOr> CpuCompiler::Compile( llvm_module->setDataLayout(jit->data_layout()); llvm_module->setTargetTriple(jit->target_triple().getTriple()); - TF_RETURN_IF_ERROR(RunHloPasses(module.get())); + TF_RETURN_IF_ERROR(RunHloPasses(module.get(), /*is_aot_compile=*/false)); HloComputation* computation = module->entry_computation(); std::unordered_map hlo_to_profile_idx; @@ -466,8 +471,8 @@ StatusOr> CpuCompiler::Compile( // ownership is std::moved. const bool embed_ir_in_executable = module->config().debug_options().xla_embed_ir_in_executable(); - const string dump_debug_json_to = - module->config().debug_options().xla_dump_debug_json_to(); + const string xla_dump_hlo_proto_to = + module->config().debug_options().xla_dump_hlo_proto_to(); if (options::CpuParallelBackendRequested(module->config())) { VLOG(1) << "Using parallel cpu backend"; @@ -487,10 +492,10 @@ StatusOr> CpuCompiler::Compile( // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); - if (!dump_debug_json_to.empty()) { + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } // If we are using the parallel CPU backend, we need to create map from @@ -594,12 +599,11 @@ StatusOr> CpuCompiler::Compile( // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); - if (!dump_debug_json_to.empty()) { + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } - // Each computation is a single function. Emit all embedded computations // before the entry computation. The order of computations returned from // GetEmbeddedComputations guarantees that a called computation occurs @@ -749,7 +753,13 @@ CpuCompiler::CompileAheadOfTime(std::vector> modules, HloModule* module = modules[i].get(); VLOG(1) << "Compiling ahead-of-time: " << module->name(); - TF_RETURN_IF_ERROR(RunHloPasses(module)); + VLOG(2) << "Before optimization:"; + XLA_VLOG_LINES(2, module->ToString()); + + TF_RETURN_IF_ERROR(RunHloPasses(module, /*is_aot_compile=*/true)); + + VLOG(2) << "After optimization:"; + XLA_VLOG_LINES(2, module->ToString()); TF_ASSIGN_OR_RETURN( SequentialHloOrdering::HloModuleSequence module_sequence, @@ -766,12 +776,12 @@ CpuCompiler::CompileAheadOfTime(std::vector> modules, // print one ourselves. XLA_VLOG_LINES(2, assignment->ToString()); - const string dump_debug_json_to = - module->config().debug_options().xla_dump_debug_json_to(); - if (!dump_debug_json_to.empty()) { + const string xla_dump_hlo_proto_to = + module->config().debug_options().xla_dump_hlo_proto_to(); + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } IrEmitter ir_emitter(*module, *assignment, &llvm_module, diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h index a301d043370b2650661d371dc177ac54ba1e8a0d..d09130247421b11d6d4879466f39b89167eb9564 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.h @@ -132,7 +132,7 @@ class CpuCompiler : public LLVMCompiler { // Runs the HLO passes which are necessary for both optimizations and // correctness. - Status RunHloPasses(HloModule* module); + Status RunHloPasses(HloModule* module, bool is_aot_compile); TF_DISALLOW_COPY_AND_ASSIGN(CpuCompiler); }; diff --git a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc index 2cd0aa788057d585c2a60bd03f596b129cc53554..662ee609232f5582ce74f4f515637b2623175e94 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_parallelization_preparation.cc @@ -116,26 +116,6 @@ StatusOr ParallelizationPreparation::RunParallelTaskAssignment( // Assign parallel tasks to HLOs in entry computation. HloComputation* computation = module->entry_computation(); for (auto* instruction : computation->instructions()) { - // Currently, we do not assign parallel tasks to instructions with at least - // one of the following properties: - // *) Internal threading (library calls to kConv, kDot, and kCustomCall). - // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot). - // *) Tuple-shaped. - // TODO(b/27458679) Parallelize instructions which are skipped here. - if (instruction->opcode() == HloOpcode::kParameter || - instruction->opcode() == HloOpcode::kConstant || - instruction->opcode() == HloOpcode::kCall || - instruction->opcode() == HloOpcode::kCustomCall || - instruction->opcode() == HloOpcode::kSelectAndScatter || - (instruction->opcode() == HloOpcode::kConvolution && - PotentiallyImplementedAsEigenConvolution(*instruction)) || - PotentiallyImplementedAsEigenDot(*instruction) || - (instruction->opcode() == HloOpcode::kFusion && - instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) || - ShapeUtil::IsTuple(instruction->shape())) { - continue; - } - // Calculate target parallel task count in [1, max_parallelism_]. const int64 target_parallel_task_count = parallel_task_assignment.GetTargetParallelTaskCount(instruction); diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc index c7155b858bda5e5640e9a6719fb394ca1360d128..7908dc173d79a4a9dcb6127ac344267e27d2b5f2 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc @@ -51,6 +51,9 @@ extern const char* const kAcquireOutfeedBufferForPopulationSymbolName = "__xla_cpu_runtime_AcquireOutfeedBufferForPopulation"; extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName = "__xla_cpu_runtime_ReleaseOutfeedBufferAfterPopulation"; +extern const char* const kParallelForkJoinSymbolName = + "__xla_cpu_runtime_ParallelForkJoin"; + extern const char* const kXlaCpuRuntimeSymbolNamePrefix = "__xla_cpu_runtime_"; } // namespace runtime } // namespace cpu diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h index 29feb7267fe97f6876827b6cbfa6217a0cecf238..2ade455b8a0a43dda8c93bbb79891439da2e4f75 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h @@ -51,6 +51,7 @@ extern const char* const kAcquireInfeedBufferForDequeueSymbolName; extern const char* const kReleaseInfeedBufferAfterDequeueSymbolName; extern const char* const kAcquireOutfeedBufferForPopulationSymbolName; extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName; +extern const char* const kParallelForkJoinSymbolName; // All symbol names for XLA CPU runtime functions need to start with this // prefix. diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc new file mode 100644 index 0000000000000000000000000000000000000000..5f5803874b7886e56da47250d0dbe297f5db16c5 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.cc @@ -0,0 +1,39 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h" + +namespace xla { +namespace cpu { + +CustomCallTargetRegistry* CustomCallTargetRegistry::Global() { + static auto* registry = new CustomCallTargetRegistry; + return registry; +} + +void CustomCallTargetRegistry::Register(const std::string& symbol, + void* address) { + std::lock_guard lock(mu_); + registered_symbols_[symbol] = address; +} + +void* CustomCallTargetRegistry::Lookup(const std::string& symbol) const { + std::lock_guard lock(mu_); + auto it = registered_symbols_.find(symbol); + return it == registered_symbols_.end() ? nullptr : it->second; +} + +} // namespace cpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h new file mode 100644 index 0000000000000000000000000000000000000000..2994642356d55df26c31553ef28dc653503d05be --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h @@ -0,0 +1,74 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_ +#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_ + +// This file is depended on by kernels that have to build for mobile devices. +// For this reason, we avoid relying on TensorFlow and instead only use the +// standard C++ library. + +#include // NOLINT +#include +#include + +namespace xla { +namespace cpu { + +// The CPU JIT compiler uses this registry to resolve symbolic CustomCall +// targets; so when using the CPU JIT, CustomCall targets need to be registered +// here with the symbol name used in the CustomCall. +// +// The XLA AOT compiler links using a standard offline linker; so when compiling +// in AOT mode, you *also* need to make sure the name of the callee (presumably +// implemented in C++) matches up with the symbolic name used in the CustomCall. +// +// We maintain the registry in both the JIT and the AOT cases for simplicity, +// but we only use it when running in JIT mode. +class CustomCallTargetRegistry { + public: + static CustomCallTargetRegistry* Global(); + + void Register(const std::string& symbol, void* address); + void* Lookup(const std::string& symbol) const; + + private: + std::unordered_map registered_symbols_; + mutable std::mutex mu_; +}; + +class RegisterCustomCallTarget { + public: + explicit RegisterCustomCallTarget(const std::string& name, void* address) { + CustomCallTargetRegistry::Global()->Register(name, address); + } +}; + +#define REGISTER_CUSTOM_CALL_CONCAT(a, b) a##b + +#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, counter) \ + static ::xla::cpu::RegisterCustomCallTarget REGISTER_CUSTOM_CALL_CONCAT( \ + custom_call_target_register, counter)(symbol, \ + reinterpret_cast(address)) + +#define REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(symbol, address) \ + REGISTER_CUSTOM_CALL_TARGET_WITH_SYM_HELPER(symbol, address, __COUNTER__) + +#define REGISTER_CUSTOM_CALL_TARGET(function) \ + REGISTER_CUSTOM_CALL_TARGET_WITH_SYM(#function, function) + +} // namespace cpu +} // namespace xla + +#endif // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_CUSTOM_CALL_TARGET_REGISTRY_H_ diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc index d3b94d75411218346cd25b0d3ecc3a9f30b56ba3..e57d49172b18beb75cfbb482c5d732ef679ebe41 100644 --- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc @@ -63,7 +63,7 @@ DotOpEmitter::DotOpEmitter(const HloInstruction& dot, bool transpose_lhs, llvm::Value* executable_run_options_value, llvm::IRBuilder<>* ir_builder, const HloModuleConfig& hlo_module_config) { PrimitiveType type = target_array.GetShape().element_type(); - TF_RET_CHECK(F32 == type || F64 == type); + TF_RET_CHECK(F32 == type || F64 == type || C64 == type); DotOpEmitter dot_emitter(dot, transpose_lhs, transpose_rhs, target_array, lhs_array, rhs_array, executable_run_options_value, ir_builder, hlo_module_config); @@ -176,7 +176,7 @@ tensorflow::Status DotOpEmitter::Emit() { llvm::BasicBlock* preheader_bb = reduction_loop->GetPreheaderBasicBlock(); ir_builder_->SetInsertPoint(preheader_bb->getTerminator()); - ir_builder_->CreateStore(llvm::ConstantFP::get(accum_type, 0.0), + ir_builder_->CreateStore(llvm::Constant::getNullValue(accum_type), accum_address); // Body basic block of reduction loop: @@ -191,9 +191,29 @@ tensorflow::Status DotOpEmitter::Emit() { llvm::Value* rhs_element = rhs_array_.EmitReadArrayElement(rhs_index, ir_builder_); - llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element); llvm::Value* accum = ir_builder_->CreateLoad(accum_address); - llvm::Value* updated_accum = ir_builder_->CreateFAdd(accum, product); + llvm::Value* updated_accum; + if (ShapeUtil::ElementIsComplex(lhs_shape)) { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + llvm::Value* product_real = ir_builder_->CreateFSub( + ir_builder_->CreateFMul(real(lhs_element), real(rhs_element)), + ir_builder_->CreateFMul(imag(lhs_element), imag(rhs_element))); + llvm::Value* product_imag = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_element), imag(rhs_element)), + ir_builder_->CreateFMul(imag(lhs_element), real(rhs_element))); + updated_accum = ir_builder_->CreateInsertValue( + accum, ir_builder_->CreateFAdd(real(accum), product_real), {0}); + updated_accum = ir_builder_->CreateInsertValue( + updated_accum, ir_builder_->CreateFAdd(imag(accum), product_imag), {1}); + } else { + llvm::Value* product = ir_builder_->CreateFMul(lhs_element, rhs_element); + updated_accum = ir_builder_->CreateFAdd(accum, product); + } ir_builder_->CreateStore(updated_accum, accum_address); // Exit basic block of reduction loop. @@ -230,11 +250,28 @@ tensorflow::Status DotOpEmitter::Emit() { tensorflow::Status DotOpEmitter::EmitScalarDot() { // A scalar dot is just a scalar multiply. + llvm::Value* result; llvm::Value* lhs_value = lhs_array_.EmitReadArrayElement(/*index=*/{}, ir_builder_); llvm::Value* rhs_value = rhs_array_.EmitReadArrayElement(/*index=*/{}, ir_builder_); - llvm::Value* result = ir_builder_->CreateFMul(lhs_value, rhs_value); + if (ShapeUtil::ElementIsComplex(lhs_array_.GetShape())) { +#define REAL(x) ir_builder_->CreateExtractValue(x, {0}) +#define IMAG(x) ir_builder_->CreateExtractValue(x, {1}) + llvm::Value* real = ir_builder_->CreateFSub( + ir_builder_->CreateFMul(REAL(lhs_value), REAL(rhs_value)), + ir_builder_->CreateFMul(IMAG(lhs_value), IMAG(rhs_value))); + llvm::Value* imag = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(REAL(lhs_value), IMAG(rhs_value)), + ir_builder_->CreateFMul(IMAG(lhs_value), REAL(rhs_value))); +#undef IMAG +#undef REAL + result = llvm::ConstantAggregateZero::get(lhs_array_.GetElementLlvmType()); + result = ir_builder_->CreateInsertValue(result, real, {0}); + result = ir_builder_->CreateInsertValue(result, imag, {1}); + } else { + result = ir_builder_->CreateFMul(lhs_value, rhs_value); + } target_array_.EmitWriteArrayElement(/*index=*/{}, result, ir_builder_); return tensorflow::Status::OK(); } diff --git a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc index 73e039250ba62b1313c98965421f6d823ca6a3b0..ba693ec89ab7c4090f8c9d1e4d65f17a80d0ac55 100644 --- a/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.cc @@ -46,8 +46,8 @@ StatusOr CpuElementalIrEmitter::EmitFloatUnaryOp( } // Create function type for the function. llvm::FunctionType* function_type = llvm::FunctionType::get( - llvm_ir::PrimitiveTypeToIrType(element_type, ir_builder_), - llvm_ir::PrimitiveTypeToIrType(element_type, ir_builder_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), /*isVarArg=*/false); // Create function declaration for 'tanhf'. llvm::Function* function = diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc index ea5b6ca4ebfd8d67681da48b0c43a95ca3685a8e..b99b36a55eee40bc66dcb1b7b1a464bf764ef0ea 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc @@ -41,6 +41,12 @@ bool PotentiallyImplementedAsEigenConvolution( ShapeUtil::HasZeroElements(kernel_shape)) { return false; } + // TODO(b/65408531): Explore using Eigen dot for complex64 type. + if (ShapeUtil::ElementIsComplex(input_shape) || + ShapeUtil::ElementIsComplex(kernel_shape)) { + return false; + } + const ConvolutionDimensionNumbers& dnums = convolution.convolution_dimension_numbers(); // Only 1D and 2D convolutions are supported at the moment. @@ -117,8 +123,9 @@ bool PotentiallyImplementedAsEigenDot(const HloInstruction& hlo) { if (hlo.opcode() == HloOpcode::kFusion && hlo.fusion_kind() == HloInstruction::FusionKind::kTransposeDot && hlo.fused_expression_root()->opcode() == HloOpcode::kDot) { - const Shape& lhs_shape = hlo.operand(0)->shape(); - const Shape& rhs_shape = hlo.operand(1)->shape(); + auto* dot = hlo.fused_expression_root(); + const Shape& lhs_shape = dot->operand(0)->shape(); + const Shape& rhs_shape = dot->operand(1)->shape(); if (ShapeUtil::HasZeroElements(lhs_shape) || ShapeUtil::HasZeroElements(rhs_shape)) { return false; diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 3d2d0f1029371159fa786f6e4463310e5e48843b..a20ce6826ca0a86f8c0d441c1e89f091cfb434f1 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -42,6 +42,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/dot_op_emitter.h" #include "tensorflow/compiler/xla/service/cpu/elemental_ir_emitter.h" #include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h" +#include "tensorflow/compiler/xla/service/cpu/shape_partition.h" #include "tensorflow/compiler/xla/service/cpu/simple_orc_jit.h" #include "tensorflow/compiler/xla/service/elemental_ir_emitter.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" @@ -186,20 +187,9 @@ void IrEmitter::InitializeIrFunction(const string& function_name) { // Even though the type of params and temps is void** in the host's view, in // LLVM IR this is represented by i8*, similarly to void*. It's up to the code // to use GEPs to unravel the indirection layers. - llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext()); - llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo(); - llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext()); - std::vector compute_function_params( - {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type}); - if (IsParallelContext()) { - compute_function_params.push_back(i64_ptr_type); - } - if (hlo_to_profile_idx_) { - compute_function_params.push_back(i64_ptr_type); - } llvm::FunctionType* compute_function_type = llvm::FunctionType::get( /*Result=*/llvm::Type::getVoidTy(module_->getContext()), - /*Params=*/compute_function_params, + /*Params=*/GetComputeFunctionParams(), /*isVarArg=*/false); // Functions with local linkage get an inlining bonus. Because we know @@ -221,7 +211,7 @@ void IrEmitter::InitializeIrFunction(const string& function_name) { (++arg_iter)->setName("run_options"); (++arg_iter)->setName("params"); (++arg_iter)->setName("temps"); - if (IsParallelContext()) { + if (num_dynamic_loop_bounds_ > 0) { (++arg_iter)->setName("dynamic_loop_bounds"); } if (hlo_to_profile_idx_) { @@ -272,9 +262,9 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } -Status IrEmitter::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status IrEmitter::HandleConstant(HloInstruction* constant) { VLOG(2) << "HandleConstant: " << constant->ToString(); + const Literal& literal = constant->literal(); llvm::GlobalVariable* global_for_const; // We avoid creating large constants in the LLVM IR since LLVM is not @@ -298,7 +288,7 @@ Status IrEmitter::HandleConstant(HloInstruction* constant, MinimumAlignmentForShape(literal.shape())); } else { llvm::Constant* initializer = - llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_); + llvm_ir::ConvertLiteralToIrConstant(literal, module_); global_for_const = new llvm::GlobalVariable( /*Module=*/*module_, /*Type=*/initializer->getType(), @@ -402,29 +392,30 @@ void IrEmitter::AttachDereferenceableMetadataForLoad(llvm::LoadInst* load, } } -Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) { // A tuple is an array of pointers, one for each operand. Each pointer points // to the output buffer of its corresponding operand. A GetTupleElement // instruction forwards a pointer to the tuple element buffer at the given // index. + auto operand = get_tuple_element->operand(0); const Shape& shape = get_tuple_element->shape(); emitted_value_[get_tuple_element] = llvm_ir::EmitGetTupleElement( shape, get_tuple_element->tuple_index(), MinimumAlignmentForShape(shape), - GetEmittedValueFor(operand), &ir_builder_); + GetEmittedValueFor(operand), &ir_builder_, module_); return Status::OK(); } -Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { +Status IrEmitter::HandleSelect(HloInstruction* select) { + auto pred = select->operand(0); + auto on_true = select->operand(1); + auto on_false = select->operand(2); TF_RET_CHECK(pred->shape().element_type() == PRED); if (ShapeUtil::IsTuple(select->shape())) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(select)); - llvm_ir::EmitTupleSelect(GetIrArrayFor(select), GetIrArrayFor(pred), - GetEmittedValueFor(on_true), - GetEmittedValueFor(on_false), &ir_builder_); + llvm_ir::EmitTupleSelect( + GetIrArrayFor(select), GetIrArrayFor(pred), GetEmittedValueFor(on_true), + GetEmittedValueFor(on_false), &ir_builder_, module_); return Status::OK(); } @@ -469,7 +460,8 @@ Status IrEmitter::HandleInfeed(HloInstruction* infeed) { tuple_element_addresses.push_back(tuple_element_address); } - llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_); + llvm_ir::EmitTuple(infeed_array, tuple_element_addresses, &ir_builder_, + module_); } else { TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kInfeed, shape, GetEmittedValueFor(infeed))); @@ -572,7 +564,7 @@ Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) { ShapeUtil::GetTupleElementShape(operand_shape, i); llvm::Value* tuple_element = llvm_ir::EmitGetTupleElement( tuple_element_shape, i, MinimumAlignmentForShape(tuple_element_shape), - value, &ir_builder_); + value, &ir_builder_, module_); TF_RETURN_IF_ERROR(EmitXfeedTransfer(XfeedKind::kOutfeed, tuple_element_shape, tuple_element)); } @@ -580,27 +572,24 @@ Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) { return Status::OK(); } -Status IrEmitter::HandleSort(HloInstruction* sort, HloInstruction* operand) { +Status IrEmitter::HandleSort(HloInstruction* sort) { // TODO(b/26783907): Implement sort on CPU. return Unimplemented("Sort is not supported on CPU (b/26783907)."); } -Status IrEmitter::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitter::HandleTuple(HloInstruction* tuple) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(tuple)); std::vector base_ptrs; - for (auto operand : operands) { + for (auto operand : tuple->operands()) { base_ptrs.push_back(GetEmittedValueFor(operand)); } - llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_); + llvm_ir::EmitTuple(GetIrArrayFor(tuple), base_ptrs, &ir_builder_, module_); return Status::OK(); } -Status IrEmitter::HandleMap( - HloInstruction* map, tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice /*static_operands*/) { +Status IrEmitter::HandleMap(HloInstruction* map) { + tensorflow::gtl::ArraySlice operands(map->operands()); + HloComputation* function = map->to_apply(); // The called computation should have been emitted previously. llvm::Function* mapped_ir_function = FindOrDie(emitted_functions_, function); @@ -617,10 +606,10 @@ Status IrEmitter::HandleMap( }); } -Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, - const Window& window, - HloComputation* function) { +Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window) { + auto operand = reduce_window->operand(0); + const Window& window = reduce_window->window(); + HloComputation* function = reduce_window->to_apply(); TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*reduce_window, /*operands=*/{operand}, /*supported_types=*/{F32})); @@ -654,7 +643,7 @@ Status IrEmitter::HandleReduceWindow(HloInstruction* reduce_window, // the initial value on the reduce_window. PrimitiveType operand_element_type = operand->shape().element_type(); llvm::Value* accumulator_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_), "reduce_window_accumulator_address", &ir_builder_, MinimumAlignmentForPrimitiveType(operand_element_type)); ir_builder_.CreateStore(ir_builder_.CreateLoad(GetEmittedValueFor( @@ -779,7 +768,7 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) { // Allocate space to keep the currently selected value, its index, and // the boolean initialized_flag, which is initially set to false. llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_), "selected_value_address", &ir_builder_, MinimumAlignmentForPrimitiveType(operand_element_type)); llvm::Value* selected_index_address = @@ -861,8 +850,8 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) { // If the 'select' function returns false, update the selected value and the // index to the currently visiting operand. llvm::Value* cond = ir_builder_.CreateICmpNE( - result, llvm::ConstantInt::get( - llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), 0), + result, + llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0), "boolean_predicate"); llvm_ir::LlvmIfData if_select_lhs = llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_); @@ -901,11 +890,12 @@ Status IrEmitter::HandleSelectAndScatter(HloInstruction* select_and_scatter) { return Status::OK(); } -Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) { +Status IrEmitter::HandleDot(HloInstruction* dot) { + auto lhs = dot->operand(0); + auto rhs = dot->operand(1); TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*dot, /*operands=*/{lhs, rhs}, - /*supported_types=*/{F32, F64})); + /*supported_types=*/{F32, F64, C64})); llvm_ir::IrArray lhs_array(GetIrArrayFor(lhs)); llvm_ir::IrArray rhs_array(GetIrArrayFor(rhs)); @@ -928,12 +918,13 @@ Status IrEmitter::HandleDot(HloInstruction* dot, HloInstruction* lhs, hlo_module_config_); } -Status IrEmitter::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs, HloInstruction* rhs, - const Window& window) { +Status IrEmitter::HandleConvolution(HloInstruction* convolution) { + auto lhs = convolution->operand(0); + auto rhs = convolution->operand(1); + const auto& window = convolution->window(); TF_RETURN_IF_ERROR(ElementTypesSameAndSupported( /*instruction=*/*convolution, /*operands=*/{lhs, rhs}, - /*supported_types=*/{F32})); + /*supported_types=*/{F32, C64})); const ConvolutionDimensionNumbers& dnums = convolution->convolution_dimension_numbers(); @@ -1089,7 +1080,7 @@ Status IrEmitter::HandleConvolution(HloInstruction* convolution, // the output entry at the given index. PrimitiveType lhs_element_type = lhs->shape().element_type(); llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(lhs_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(lhs_element_type, module_), "convolution_sum_address", &ir_builder_, MinimumAlignmentForPrimitiveType(lhs_element_type)); ir_builder_.CreateStore( @@ -1305,14 +1296,14 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) { PrimitiveType element_type = operand->shape().element_type(); // Used to calculate E(X). llvm::Value* sum_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), "sum_address", &ir_builder_, MinimumAlignmentForPrimitiveType(element_type)); // Used to calculate E(X^2). llvm::Value* sum_square_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(element_type, module_), "sum_square_address", &ir_builder_, MinimumAlignmentForPrimitiveType(element_type)); @@ -1435,7 +1426,7 @@ Status IrEmitter::HandleBatchNormTraining(HloInstruction* batch_norm_training) { .EmitLoop(IrName(batch_norm_training, "normalize"))); llvm_ir::EmitTuple(GetIrArrayFor(batch_norm_training), - {normalized, mean, var}, &ir_builder_); + {normalized, mean, var}, &ir_builder_, module_); return Status::OK(); } @@ -1498,6 +1489,14 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator( } const Shape& root_shape = root_instruction->shape(); + if (ShapeUtil::ElementIsComplex(root_shape)) { + // TODO(b/65408531): Complex add could by done via bitcast to + // Complex multiply would be more challenging. We could perhaps use a + // strided load to get all reals in a vector, all imags in a vector, or use + // CreateShuffleVector on a bitcast to float x [2N]. + *failure_reason = "complex values not supported"; + return nullptr; + } bool root_is_floating_point = ShapeUtil::ElementIsFloating(root_shape); bool root_is_integral = ShapeUtil::ElementIsIntegral(root_shape); bool root_is_signed = ShapeUtil::ElementIsSigned(root_shape); @@ -1519,7 +1518,7 @@ IrEmitter::ReductionGenerator IrEmitter::MatchReductionGenerator( // This is visually similar to ElementalIrEmitter, though conceptually we're // doing something different here. ElementalIrEmitter emits scalar operations // while these emit scalar or vector operations depending on the type of the - // operands. + // operands. See CreateShardedVectorType for the actual types in use here. switch (root_instruction->opcode()) { default: *failure_reason = "did not recognize root instruction opcode"; @@ -1596,7 +1595,7 @@ IrEmitter::ShardedVectorType IrEmitter::CreateShardedVectorType( ShardedVectorType sharded_vector_type; llvm::Type* element_ir_type = - llvm_ir::PrimitiveTypeToIrType(element_type, &ir_builder_); + llvm_ir::PrimitiveTypeToIrType(element_type, module_); for (int i = 0, e = 1 + tensorflow::Log2Ceiling(element_count); i < e; i++) { // For every power of two present in element_count, we generate one or more @@ -1901,10 +1900,11 @@ StatusOr IrEmitter::EmitVectorizedReduce( return true; } -Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) { +Status IrEmitter::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->mutable_operand(0); + auto init_value = reduce->mutable_operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); if (!options::VectorizedReduceDisabled(hlo_module_config_)) { string vectorization_failure_reason; TF_ASSIGN_OR_RETURN( @@ -1929,7 +1929,7 @@ Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, // Initialize an accumulator with init_value. PrimitiveType accumulator_type = reduce->shape().element_type(); llvm::AllocaInst* accumulator_addr = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(accumulator_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(accumulator_type, module_), "accumulator", &ir_builder_, MinimumAlignmentForPrimitiveType(accumulator_type)); llvm::Value* init_value_addr = GetEmittedValueFor(init_value); @@ -1983,9 +1983,9 @@ Status IrEmitter::HandleSend(HloInstruction* send) { return Unimplemented("Send is not implemented on CPU. See b/33942983."); } -Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) { +Status IrEmitter::HandleSlice(HloInstruction* slice) { VLOG(2) << "HandleSlice: " << slice->ToString(); - + auto operand = slice->operand(0); // The code below emits a sequential loop nest. For the parallel backend, use // EmitParallelTargetElementLoop() which respects dynamic loop bounds. if (ShouldEmitParallelLoopFor(*slice)) { @@ -2118,20 +2118,17 @@ Status IrEmitter::HandleSlice(HloInstruction* slice, HloInstruction* operand) { return Status::OK(); } -Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* /*start_indices*/) { +Status IrEmitter::HandleDynamicSlice(HloInstruction* dynamic_slice) { if (ShapeUtil::IsScalar(dynamic_slice->shape())) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_slice)); - return EmitMemcpy(*operand, *dynamic_slice); + return EmitMemcpy(*dynamic_slice->operand(0), *dynamic_slice); } return DefaultAction(dynamic_slice); } -Status IrEmitter::HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) { +Status IrEmitter::HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) { + auto update = dynamic_update_slice->operand(1); if (ShapeUtil::IsScalar(dynamic_update_slice->shape())) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(dynamic_update_slice)); return EmitMemcpy(*update, *dynamic_update_slice); @@ -2258,6 +2255,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { return Status::OK(); } else if (llvm_ir::CanEmitFusedDynamicUpdateSliceInPlace(fusion, assignment_)) { + VLOG(3) << "HandleFusion FusedDynamicUpdateSliceInPlace"; CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_); TF_RETURN_IF_ERROR(EmitTargetAddressForOp(fusion)); @@ -2267,6 +2265,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { fusion, operands, GetIrArrayFor(fusion), &elemental_emitter, &ir_builder_); } else if (fusion->fusion_kind() == HloInstruction::FusionKind::kLoop) { + VLOG(3) << "HandleFusion kLoop"; CpuElementalIrEmitter elemental_emitter(hlo_module_config_, this, module_); auto operands = GetIrArraysForOperandsOf(fusion); FusedIrEmitter fused_emitter(operands, &elemental_emitter); @@ -2288,15 +2287,26 @@ Status IrEmitter::HandleCall(HloInstruction* call) { } TF_RETURN_IF_ERROR(EmitTargetAddressForOp(call)); - EmitArrayFunctionCallInto(call_ir_function, parameter_addresses, - emitted_value_[call], computation->name()); + + if (!computation->root_instruction()->outer_dimension_partitions().empty() && + !parallel_cpu_backend_) { + // ParallelTaskAssignment assigned partitions, emit call to + // ParallelForkJoin. + TF_RETURN_IF_ERROR(EmitParallelForkJoin(parameter_addresses, + emitted_value_[call], computation, + call_ir_function)); + } else { + EmitArrayFunctionCallInto(call_ir_function, parameter_addresses, + emitted_value_[call], computation->name()); + } + return Status::OK(); } -Status IrEmitter::HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) { +Status IrEmitter::HandleCustomCall(HloInstruction* custom_call) { + tensorflow::gtl::ArraySlice operands( + custom_call->operands()); + tensorflow::StringPiece custom_call_target(custom_call->custom_call_target()); llvm::Type* i8_ptr_type = ir_builder_.getInt8PtrTy(); llvm::AllocaInst* operands_alloca = llvm_ir::EmitAllocaAtFunctionEntryWithCount( @@ -2399,8 +2409,7 @@ Status IrEmitter::HandleWhile(HloInstruction* xla_while) { {while_result}, IrName(xla_while, "cond")); llvm::Value* while_predicate = ir_builder_.CreateICmpNE( while_condition, - llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), - 0)); + llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType(PRED, module_), 0)); // Branches to the body or to the while exit depending on the condition. llvm::BasicBlock* body_bb = llvm::BasicBlock::Create( @@ -2541,7 +2550,7 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source, unsigned element_alignment = GCD( primitive_type_size, MinimumAlignmentForPrimitiveType(primitive_type)); llvm::Type* primitive_ptr_type = llvm::PointerType::getUnqual( - llvm_ir::PrimitiveTypeToIrType(primitive_type, &ir_builder_)); + llvm_ir::PrimitiveTypeToIrType(primitive_type, module_)); if (element_count == 1) { auto* load_instruction = ir_builder_.CreateAlignedLoad( @@ -2567,9 +2576,9 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source, } } -Status IrEmitter::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitter::HandleConcatenate(HloInstruction* concatenate) { + tensorflow::gtl::ArraySlice operands( + concatenate->operands()); string failure_reason; TF_ASSIGN_OR_RETURN( bool successful, @@ -2599,7 +2608,7 @@ Status IrEmitter::FinishVisit(HloInstruction* root) { // For the parallel cpu backend, we record the total for each embedded // computation callee with its caller kCall HLO. HloInstruction* hlo_to_lookup = nullptr; - if (IsParallelContext()) { + if (parallel_cpu_backend_ && is_top_level_computation_) { auto* computation = root->parent(); auto* entry_computation = computation->parent()->entry_computation(); if (computation != entry_computation) { @@ -2754,7 +2763,22 @@ llvm::Value* IrEmitter::GetEmittedValueFor(const HloInstruction* hlo) { } llvm::Type* IrEmitter::IrShapeType(const Shape& shape) { - return llvm_ir::ShapeToIrType(shape, &ir_builder_); + return llvm_ir::ShapeToIrType(shape, module_); +} + +std::vector IrEmitter::GetComputeFunctionParams() { + llvm::Type* i8_ptr_type = llvm::Type::getInt8PtrTy(module_->getContext()); + llvm::Type* i8_ptr_ptr_type = i8_ptr_type->getPointerTo(); + llvm::Type* i64_ptr_type = llvm::Type::getInt64PtrTy(module_->getContext()); + std::vector compute_function_params( + {i8_ptr_type, i8_ptr_type, i8_ptr_ptr_type, i8_ptr_ptr_type}); + if (num_dynamic_loop_bounds_ > 0) { + compute_function_params.push_back(i64_ptr_type); + } + if (hlo_to_profile_idx_) { + compute_function_params.push_back(i64_ptr_type); + } + return compute_function_params; } llvm::Argument* IrEmitter::GetResultArgument() { @@ -2762,7 +2786,7 @@ llvm::Argument* IrEmitter::GetResultArgument() { } llvm::Argument* IrEmitter::GetProfileCountersArgument() { - const int64 arg_index = IsParallelContext() ? 5 : 4; + const int64 arg_index = num_dynamic_loop_bounds_ > 0 ? 5 : 4; return hlo_to_profile_idx_ ? GetArg(compute_function_, arg_index) : nullptr; } @@ -2845,18 +2869,11 @@ llvm::Value* IrEmitter::EmitElementFunctionCall( AsStringRef(tensorflow::strings::StrCat(name, "_return_value"))); } -// Emits a core function call based on the following pseudo-code. -// -// char** parameter_addresses_buffer = -// allocate buffer with a pointer for each parameter to the function -// for each parameter index, i.e. for i = 0, ..., #parameters: -// parameter_addresses_buffer[i] = parameter_addresses[i] -// call function(return_value_buffer, -// parameter_addresses_buffer, -// temps) -// return return_value_buffer -- address of the return value. -void IrEmitter::EmitArrayFunctionCallInto( - llvm::Function* function, +// Emits code to allocate an array of parameter address pointers, and store +// each address from 'parameter_addresses'. +// Returns an array of compute function call arguments (including parameter +// address buffer). +std::vector IrEmitter::GetArrayFunctionCallArguments( tensorflow::gtl::ArraySlice parameter_addresses, llvm::Value* return_value_buffer, tensorflow::StringPiece name) { llvm::Value* parameter_addresses_buffer = @@ -2885,7 +2902,26 @@ void IrEmitter::EmitArrayFunctionCallInto( if (auto* profile_counters = GetProfileCountersArgument()) { arguments.push_back(profile_counters); } - ir_builder_.CreateCall(function, arguments); + return arguments; +} + +// Emits a core function call based on the following pseudo-code. +// +// char** parameter_addresses_buffer = +// allocate buffer with a pointer for each parameter to the function +// for each parameter index, i.e. for i = 0, ..., #parameters: +// parameter_addresses_buffer[i] = parameter_addresses[i] +// call function(return_value_buffer, +// parameter_addresses_buffer, +// temps) +// return return_value_buffer -- address of the return value. +void IrEmitter::EmitArrayFunctionCallInto( + llvm::Function* function, + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* return_value_buffer, tensorflow::StringPiece name) { + ir_builder_.CreateCall( + function, GetArrayFunctionCallArguments(parameter_addresses, + return_value_buffer, name)); } llvm::Value* IrEmitter::EmitArrayFunctionCall( @@ -2897,7 +2933,7 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall( PrimitiveType return_type = return_shape.element_type(); llvm::Value* return_value_buffer = llvm_ir::EmitAllocaAtFunctionEntryWithCount( - llvm_ir::PrimitiveTypeToIrType(return_type, &ir_builder_), elements, + llvm_ir::PrimitiveTypeToIrType(return_type, module_), elements, tensorflow::strings::StrCat(name, "_return_value_address"), &ir_builder_, MinimumAlignmentForPrimitiveType(return_type)); EmitArrayFunctionCallInto(function, parameter_addresses, return_value_buffer, @@ -2905,6 +2941,110 @@ llvm::Value* IrEmitter::EmitArrayFunctionCall( return return_value_buffer; } +// Emits a call to a runtime fork/join function which dispatches parallel +// calls to 'parallel_function' (and joins threads before returning). +Status IrEmitter::EmitParallelForkJoin( + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* output_address, HloComputation* computation, + llvm::Function* parallel_function) { + HloInstruction* root = computation->root_instruction(); + + // Build ParallelForkJoin function type. + std::vector compute_function_params = GetComputeFunctionParams(); + // Number of parallel compute functions. + compute_function_params.push_back(ir_builder_.getInt32Ty()); + // Array of partitions. There is an array element for each + // partition x partition_dim x 2 (for dimension start and limit). + compute_function_params.push_back( + llvm::Type::getInt64PtrTy(module_->getContext())); + // Number of partitioned most-major dimensions in 'root.shape'. + compute_function_params.push_back(ir_builder_.getInt32Ty()); + // Function pointer for compute function to be dispatched in parallel. + compute_function_params.push_back( + llvm::Type::getInt8PtrTy(module_->getContext())); + + llvm::FunctionType* fork_join_type = llvm::FunctionType::get( + /*Result=*/llvm::Type::getVoidTy(module_->getContext()), + /*Params=*/compute_function_params, + /*isVarArg=*/false); + + llvm::Function* fork_join_func = + llvm::cast(module_->getOrInsertFunction( + runtime::kParallelForkJoinSymbolName, fork_join_type)); + fork_join_func->setCallingConv(llvm::CallingConv::C); + fork_join_func->setDoesNotThrow(); + + // Add common compute function arguments. + const string name = computation->name(); + std::vector arguments = + GetArrayFunctionCallArguments(parameter_addresses, output_address, name); + + // Create ShapePartitionIterator to generate all partitions of 'root.shape'. + ShapePartitionIterator partition_iterator(root->shape(), + root->outer_dimension_partitions()); + const int64 num_partitions = partition_iterator.GetTotalPartitionCount(); + // Add argument specifying the number of parallel partitions. + arguments.push_back(ir_builder_.getInt32(num_partitions)); + + // The number of partitioned most-major dimensions in 'root.shape'. + const int32 num_partitioned_dims = root->outer_dimension_partitions().size(); + // A dimension partition consists of two elements: [start_index, limit_index). + const int32 dim_partition_size = 2; + // Calculate array partition stride. + const int32 array_partition_stride = + num_partitioned_dims * dim_partition_size; + // Calculate the total number of elements in the partition array. + const int32 partition_array_size = + dim_partition_size * num_partitioned_dims * num_partitions; + + // Store dimension partition values as llvm constants in 'partitions'. + // See comments in runtime_fork_join.cc for array layout description. + std::vector partitions(partition_array_size); + for (int32 i = 0; i < num_partitions; ++i) { + std::vector> dim_partitions = + partition_iterator.GetPartition(i); + CHECK_EQ(num_partitioned_dims, dim_partitions.size()); + const int32 partition_index = i * array_partition_stride; + for (int32 j = 0; j < num_partitioned_dims; ++j) { + const std::pair& dim_partition = dim_partitions[j]; + const int32 index = partition_index + j * dim_partition_size; + // Store partition [dim_start, dim_limit) intervals for each dimension. + partitions[index] = ir_builder_.getInt64(dim_partition.first); + partitions[index + 1] = + ir_builder_.getInt64(dim_partition.first + dim_partition.second); + } + } + + // Create global variable out of dimension partitions in 'partitions'. + llvm::ArrayType* partitions_array_type = + llvm::ArrayType::get(ir_builder_.getInt64Ty(), partition_array_size); + llvm::Constant* partitions_array = + llvm::ConstantArray::get(partitions_array_type, partitions); + llvm::GlobalVariable* global_partitions_array = new llvm::GlobalVariable( + /*Module=*/*module_, + /*Type=*/partitions_array_type, + /*isConstant=*/true, + /*Linkage=*/llvm::GlobalValue::PrivateLinkage, + /*Initializer=*/partitions_array, + /*Name=*/ + AsStringRef( + tensorflow::strings::StrCat(name, "_parallel_dimension_partitions"))); + + // Add argument specifying parallel dimension partitions. + arguments.push_back(ir_builder_.CreateBitCast( + global_partitions_array, + llvm::Type::getInt64PtrTy(module_->getContext()))); + // Add argument specifying the number of partitioned most-major dimensions. + arguments.push_back(ir_builder_.getInt32(num_partitioned_dims)); + // Add argument for parallel compute function pointer. + arguments.push_back( + ir_builder_.CreateBitCast(parallel_function, ir_builder_.getInt8PtrTy())); + // Emit call to parallel fork/join. + ir_builder_.CreateCall(fork_join_func, arguments); + + return Status::OK(); +} + Status IrEmitter::EmitTargetAddressForOp(const HloInstruction* op) { llvm::Value* addr; const Shape& target_shape = op->shape(); @@ -2968,7 +3108,7 @@ Status IrEmitter::EmitTargetElementLoop( for (int64 i = 0; i < output_arrays.size(); ++i) { tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer()); } - llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_); + llvm_ir::EmitTuple(target_array, tuple_operand_ptrs, &ir_builder_, module_); } else { if (ShouldEmitParallelLoopFor(*target_op)) { diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 53c4b6f24190d53756e1e4e46ed684dec2895f5e..5d061e11e3c9e07bdcfdc749711e4369ec2bea2a 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -154,62 +154,36 @@ class IrEmitter : public DfsHloVisitorWithDefault { Status DefaultAction(HloInstruction* hlo) override; Status HandleBitcast(HloInstruction* bitcast) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleSelect(HloInstruction* select) override; + Status HandleDot(HloInstruction* dot) override; + Status HandleConvolution(HloInstruction* convolution) override; Status HandleBatchNormTraining(HloInstruction* batch_norm_training) override; Status HandleBatchNormGrad(HloInstruction* batch_norm_grad) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; Status HandleInfeed(HloInstruction* infeed) override; Status HandleOutfeed(HloInstruction* outfeed) override; - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override; + Status HandleSort(HloInstruction* sort) override; Status HandleParameter(HloInstruction* parameter) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override; + Status HandleReduce(HloInstruction* reduce) override; + Status HandleReduceWindow(HloInstruction* reduce_window) override; Status HandleSelectAndScatter(HloInstruction* select_and_scatter) override; Status HandleSend(HloInstruction* send) override; - Status HandleSlice(HloInstruction* slice, - HloInstruction* /*operand*/) override; - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* /*operand*/, - HloInstruction* /*start_indices*/) override; - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* /*operand*/, - HloInstruction* /*update*/, - HloInstruction* /*start_indices*/) override; + Status HandleSlice(HloInstruction* slice) override; + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override; + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override; Status HandleRecv(HloInstruction* recv) override; Status HandlePad(HloInstruction* pad) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleMap(HloInstruction* map) override; Status HandleFusion(HloInstruction* fusion) override; Status HandleCall(HloInstruction* call) override; - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override; + Status HandleCustomCall(HloInstruction* custom_call) override; Status HandleWhile(HloInstruction* xla_while) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; Status FinishVisit(HloInstruction* root) override; Status Preprocess(HloInstruction* hlo) override; @@ -249,6 +223,9 @@ class IrEmitter : public DfsHloVisitorWithDefault { // Convenience function to get the IR type matching the given shape. llvm::Type* IrShapeType(const Shape& shape); + // Returns an array of compute function parameter types. + std::vector GetComputeFunctionParams(); + // Get the llvm::Value* that represents the "retval" argument of the // computation function being emitted by this emitter. llvm::Argument* GetResultArgument(); @@ -323,6 +300,18 @@ class IrEmitter : public DfsHloVisitorWithDefault { tensorflow::gtl::ArraySlice parameter_addresses, tensorflow::StringPiece name); + // Returns an array of compute function call arguments. + std::vector GetArrayFunctionCallArguments( + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* return_value_buffer, tensorflow::StringPiece name); + + // Emits a call to a runtime fork/join function which dispatches parallel + // calls to 'parallel_function' (and joins threads before returning). + Status EmitParallelForkJoin( + tensorflow::gtl::ArraySlice parameter_addresses, + llvm::Value* output_address, HloComputation* computation, + llvm::Function* parallel_function); + // Verifies that the element types of all of the given operand instructions // match and are of one of the given supported types. Status ElementTypesSameAndSupported( @@ -596,12 +585,6 @@ class IrEmitter : public DfsHloVisitorWithDefault { Status EmitXfeedTransfer(XfeedKind kind, const Shape& shape, llvm::Value* program_buffer_address); - // Returns true if the current function being emitted is called in a - // parallel context (returns false otherwise). - bool IsParallelContext() { - return parallel_cpu_backend_ && is_top_level_computation_; - } - const HloModuleConfig& hlo_module_config_; const bool parallel_cpu_backend_; diff --git a/tensorflow/compiler/xla/service/cpu/layout_assignment.cc b/tensorflow/compiler/xla/service/cpu/layout_assignment.cc index 02e691b213b3fab1b4ec69cb0ced272fbcdc51c0..c446b6b792a042da2500ea6a175fdca4c70bcab6 100644 --- a/tensorflow/compiler/xla/service/cpu/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/cpu/layout_assignment.cc @@ -108,21 +108,26 @@ Status CpuLayoutAssignment::AddBackendConstraints( constraints->SetOperandLayout(col_major_shape(rhs_shape), dot, 1)); } else if (PotentiallyImplementedAsEigenDot(*instruction)) { const HloInstruction* dot = instruction; - const HloInstruction* lhs_instruction = dot->operand(0); - const HloInstruction* rhs_instruction = dot->operand(1); - // In order to implement `dot` with Eigen dot, the layouts of the lhs, // rhs, and output need to be row-major. // // These constraints are not hard constraints. Ideally, we should decide // which layouts to choose according to some cost model. Shape output_shape(row_major_shape(dot->shape())); + + const HloInstruction* lhs_instruction = dot->operand(0); Shape lhs_shape(row_major_shape(lhs_instruction->shape())); - Shape rhs_shape(row_major_shape(rhs_instruction->shape())); + TF_RETURN_IF_ERROR(constraints->SetOperandLayout(lhs_shape, dot, 0)); + + // dot is a kDot or a kTransposeDot fusion node. In the latter case, if + // it represents X @ X, it may have just one operand. + if (dot->operand_count() > 1) { + const HloInstruction* rhs_instruction = dot->operand(1); + Shape rhs_shape(row_major_shape(rhs_instruction->shape())); + TF_RETURN_IF_ERROR(constraints->SetOperandLayout(rhs_shape, dot, 1)); + } // Set layouts of the instructions' shapes. - TF_RETURN_IF_ERROR(constraints->SetOperandLayout(lhs_shape, dot, 0)); - TF_RETURN_IF_ERROR(constraints->SetOperandLayout(rhs_shape, dot, 1)); TF_RETURN_IF_ERROR(constraints->SetInstructionLayout(output_shape, dot)); } else { for (int64 operand_no = 0; operand_no < instruction->operand_count(); diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc index d4b5e41f5090856d68135643c1d2ee94c27491db..c2213c8f2ef592c537daf9abe2ffa10b83a8fa4c 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc +++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc @@ -48,29 +48,56 @@ class SimpleCostModel : public ParallelCostModel { class DefaultCostModel : public ParallelCostModel { public: DefaultCostModel(const int64 max_parallelism, + const HloCostAnalysis::ShapeSizeFunction& shape_size, std::unique_ptr cost_analysis) : max_parallelism_(max_parallelism), + shape_size_(shape_size), cost_analysis_(std::move(cost_analysis)) {} ~DefaultCostModel() override {} int64 GetParallelTaskCount(HloInstruction* instruction) override { - // Calculate the instruction cost in cycles. - // TODO(29630486) Improve on this linear cost model. - // Consider making 'min_cost_per_thread' be a function of the target - // bandwidth limit for instructions with low arithmetic complexity. - const int64 instruction_cost = - 1 * cost_analysis_->flop_count(*instruction) + - 2 * cost_analysis_->transcendental_count(*instruction) + - 10 * cost_analysis_->bytes_accessed(*instruction); - // Minimum per-thread cost is 100us of work on a 2GHz core. - const int64 min_cost_per_thread = 100000; + // Parameters for parallel task count computation. + int64 instruction_cost; + int64 min_cost_per_thread; + int64 max_parallelism; + // Calculate flops-to-bytes-ratio for 'instruction'. + const int64 bytes_accessed = + std::max(1LL, cost_analysis_->bytes_accessed(*instruction)); + const float flops_to_bytes_ratio = + cost_analysis_->flop_count(*instruction) / + static_cast(bytes_accessed); + // Check for I/O bound instructions. + if (flops_to_bytes_ratio <= 1.0) { + // Limit max parallelism for I/O bound instructions by assuming a + // sub-linear scaling function (fit based on empirical benchmark results). + // TODO(29630486) Develop system bandwidth model. + max_parallelism = + std::ceil(std::sqrt(tensorflow::port::NumSchedulableCPUs())); + // Use shape size instruction cost and L2 cache size min per-thread cost. + instruction_cost = shape_size_(instruction->shape()); + min_cost_per_thread = 256LL << 10; // 256KB L2 Cache size. + } else { + // Use max parallelism for compute bound instructions. + max_parallelism = max_parallelism_; + // Calculate the instruction cost in cycles. + // TODO(29630486) Improve on this linear cost model. + // Consider making 'min_cost_per_thread' be a function of the target + // bandwidth limit for instructions with low arithmetic complexity. + instruction_cost = + 1 * cost_analysis_->flop_count(*instruction) + + 2 * cost_analysis_->transcendental_count(*instruction) + + 10 * cost_analysis_->bytes_accessed(*instruction); + // Minimum per-thread cost is 100us of work on a 2GHz core. + min_cost_per_thread = 100000; + } // Return target parallel task count in [1, max_parallelism_]. - return std::min(max_parallelism_, + return std::min(max_parallelism, std::max(1LL, instruction_cost / min_cost_per_thread)); } private: const int64 max_parallelism_; + const HloCostAnalysis::ShapeSizeFunction shape_size_; const std::unique_ptr cost_analysis_; }; @@ -86,7 +113,7 @@ ParallelTaskAssignment::ParallelTaskAssignment( Status status = computation->root_instruction()->Accept(cost_analysis.get()); if (status.ok()) { // Set default cost model based on 'cost_analysis'. - cost_model_.reset(new DefaultCostModel(max_parallelism, + cost_model_.reset(new DefaultCostModel(max_parallelism, shape_size, std::move(cost_analysis))); } else { // Fall back to a simple cost model based on hlo size and L2 cache size. @@ -109,6 +136,8 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount( instruction->opcode() == HloOpcode::kCall || instruction->opcode() == HloOpcode::kCustomCall || instruction->opcode() == HloOpcode::kSelectAndScatter || + instruction->opcode() == HloOpcode::kGetTupleElement || + instruction->opcode() == HloOpcode::kBitcast || (instruction->opcode() == HloOpcode::kConvolution && PotentiallyImplementedAsEigenConvolution(*instruction)) || PotentiallyImplementedAsEigenDot(*instruction) || @@ -121,5 +150,102 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount( return cost_model_->GetParallelTaskCount(instruction); } +StatusOr ParallelTaskAssigner::Run(HloModule* module) { + XLA_VLOG_LINES(2, "ParallelTaskAssigner ENTRY"); + XLA_VLOG_LINES(3, module->ToString()); + + // Compute target parallel task counts for all instructions in 'module'. + HloToParallelTasks hlo_to_parallel_tasks; + ComputeTargetParallelTasks(module, &hlo_to_parallel_tasks); + + // Assign parallel tasks to target specific instructions in 'module'. + // TODO(b/27458679) Support inter-op parallelism. + bool changed = AssignParallelTasks(module, hlo_to_parallel_tasks); + + XLA_VLOG_LINES(2, "ParallelTaskAssigner EXIT"); + XLA_VLOG_LINES(3, module->ToString()); + return changed; +} + +bool ParallelTaskAssigner::AssignParallelTasks( + HloModule* module, const HloToParallelTasks& hlo_to_parallel_tasks) { + return AssignParallelTasksHelper(module, module->entry_computation(), + hlo_to_parallel_tasks); +} + +bool ParallelTaskAssigner::AssignParallelTasksHelper( + HloModule* module, HloComputation* computation, + const HloToParallelTasks& hlo_to_parallel_tasks) { + bool changed = false; + // Snapshot set of instructions because outlining modifies the set below. + std::vector instructions(computation->instructions().begin(), + computation->instructions().end()); + for (auto* instruction : instructions) { + // Assign parallel tasks to sub-computations for While and Call HLOs. + // TODO(b/27458679) Evaluate alternative intra-op parallelsim placement, + // and support other callable computations like reduce. + if (instruction->opcode() == HloOpcode::kWhile) { + changed |= AssignParallelTasksHelper(module, instruction->while_body(), + hlo_to_parallel_tasks); + continue; + } else if (instruction->opcode() == HloOpcode::kCall) { + changed |= AssignParallelTasksHelper(module, instruction->to_apply(), + hlo_to_parallel_tasks); + continue; + } + // Skip if no parallel tasks were computed in first pass. + auto it = hlo_to_parallel_tasks.find(instruction); + if (it == hlo_to_parallel_tasks.end()) { + continue; + } + // Get target parallel task count computed for 'instruction'. + const int64 target_parallel_task_count = (*it).second; + // Assign feasible dimension partitions (based on actual dimension sizes). + auto dim_partition_counts = ShapePartitionAssigner(instruction->shape()) + .Run(target_parallel_task_count); + const int64 total_partition_count = + ShapePartitionAssigner::GetTotalPartitionCount(dim_partition_counts); + if (total_partition_count <= 1) { + // Feasible partition calculation resulting in no partitioning, so skip. + continue; + } + + // Outline 'instruction' in 'computation' for parallel task assignment. + auto* call = module->OutlineExpressionFromComputation( + {instruction}, + tensorflow::strings::StrCat("parallel_", instruction->name()), + computation); + + // Set assigned dimension partitioning to 'instruction'. + auto* new_root = call->to_apply()->root_instruction(); + new_root->set_outer_dimension_partitions(dim_partition_counts); + + VLOG(2) << "Assigned parallel task count: " << total_partition_count + << " to instruction: " << new_root->name() + << " parent: " << new_root->parent()->name(); + changed = true; + } + return changed; +} + +void ParallelTaskAssigner::ComputeTargetParallelTasks( + HloModule* module, HloToParallelTasks* hlo_to_parallel_tasks) { + // Compute parallel task counts for all instructions in 'module'. + for (auto* computation : module->computations()) { + if (computation->IsFusionComputation()) { + continue; + } + for (auto* instruction : computation->instructions()) { + // Query ParallelTaskAssignment for target parallel task count. + const int64 target_parallel_task_count = + parallel_task_assignment_.GetTargetParallelTaskCount(instruction); + if (target_parallel_task_count > 1) { + hlo_to_parallel_tasks->insert( + {instruction, target_parallel_task_count}); + } + } + } +} + } // namespace cpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h index 15f065a3ad44b39819a62bc0447785596a3bd29c..e036da5784f6151eb3b01107ec7f3ab820071a60 100644 --- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h +++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h" #include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" namespace xla { namespace cpu { @@ -49,6 +50,54 @@ class ParallelTaskAssignment { std::unique_ptr cost_model_; }; +// ParallelTaskAssigner computes target parallel task counts for all HLOs +// in the module, then assigns parallel task counts to HLOs in the entry +// computation, or to HLOs in embedded computations invoked by (potentially +// nested) kWhile or kCall instructions. +// Each HLO which is assigned parallel task counts is outlined into its +// own embedded computation, which is compiled as a parallel compute function, +// and which is invoked from a kCall instruction that is lowered in codegen to +// a runtime parallel fork/join call. +class ParallelTaskAssigner : public HloPassInterface { + public: + // 'max_parallelism': the maximum parallel task count per instruction. + // 'shape_size': shape size function used by HloCostAnalysis during parallel + // task assignment. + // 'module': the containing HloModule. + ParallelTaskAssigner(const int64 max_parallelism, + const HloCostAnalysis::ShapeSizeFunction& shape_size, + HloModule* module) + : parallel_task_assignment_(max_parallelism, shape_size, module) {} + ~ParallelTaskAssigner() override {} + + tensorflow::StringPiece name() const override { + return "cpu-parallel-task-assigner"; + } + + // Run parallel task assigner on 'module'. + // Returns true if the computation was changed, false otherwise. + StatusOr Run(HloModule* module) override; + + private: + using HloToParallelTasks = std::unordered_map; + + // Assigns target parallel tasks from 'hlo_to_parallel_tasks' to HLOs in + // 'module'. + // Returns true if the computation was changed, false otherwise. + bool AssignParallelTasks(HloModule* module, + const HloToParallelTasks& hlo_to_parallel_tasks); + bool AssignParallelTasksHelper( + HloModule* module, HloComputation* computation, + const HloToParallelTasks& hlo_to_parallel_tasks); + + // Computes target parallel task counts (returned in 'parallel_task_counts') + // for parallelizable instructions in 'module'. + void ComputeTargetParallelTasks(HloModule* module, + HloToParallelTasks* hlo_to_parallel_tasks); + + ParallelTaskAssignment parallel_task_assignment_; +}; + } // namespace cpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc new file mode 100644 index 0000000000000000000000000000000000000000..d03da46575b331de113cc5f33c2b4267504e8308 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.cc @@ -0,0 +1,97 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h" + +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/compiler/xla/executable_run_options.h" +#include "tensorflow/core/lib/core/blocking_counter.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +using tensorflow::int32; +using tensorflow::int64; +using tensorflow::uint64; + +using ComputeFunctionType = void (*)(void*, const void*, const void**, void**, + int64*, uint64*); + +// Dispatches 'num_partitions - 1' calls to 'function_ptr' in parallel. +// Calls 'function_ptr' for first partition inline. +// Uses blocking counter to synchonize threads after parallel calls complete. +// +// The 'partitions' array has a total number of elements equal to +// 'num_partitions * num_partitioned_dims * 2' (the '2' is necessary to specify +// dimension start and limit indices). +// +// The 'partitions' array layout stores array elements in memory with dimension +// start limit as the most-minor dimension, followed by dimension, then +// partition. +// +// EX: Layout of 'partitions' array with 'num_partitions = 2', and +// 'num_partitioned_dims = 3' +// +// [partition0_dim0_start] +// [partition0_dim0_limit] +// [partition0_dim1_start] +// [partition0_dim1_limit] +// [partition0_dim2_start] +// [partition0_dim2_limit] +// [partition1_dim0_start] +// [partition1_dim0_limit] +// [partition1_dim1_start] +// [partition1_dim1_limit] +// [partition1_dim2_start] +// [partition1_dim2_limit] +// +void __xla_cpu_runtime_ParallelForkJoin( + void* result_ptr, const void* run_options_ptr, const void** params, + void** temps, uint64* prof_counters, int32 num_partitions, + int64* partitions, int32 num_partitioned_dims, void* function_ptr) { + VLOG(2) << "ParallelForkJoin ENTRY" + << " num_partitions: " << num_partitions + << " num_partitioned_dims: " << num_partitioned_dims; + CHECK_GT(num_partitions, 1); + CHECK_GT(num_partitioned_dims, 0); + const xla::ExecutableRunOptions* run_options = + static_cast(run_options_ptr); + ComputeFunctionType function = + reinterpret_cast(function_ptr); + // Compute partition stride in 'partitions' array. + const int64 stride = 2 * num_partitioned_dims; + + // Dispatch 'num_partitions - 1' compute functions to run in parallel. + tensorflow::BlockingCounter bc(num_partitions - 1); + for (int32 i = 1; i < num_partitions; ++i) { + const int64 offset = i * stride; + run_options->intra_op_thread_pool()->enqueueNoNotification( + [i, function, result_ptr, run_options_ptr, params, temps, prof_counters, + partitions, offset, &bc]() { + function(result_ptr, run_options_ptr, params, temps, + &partitions[offset], prof_counters); + bc.DecrementCount(); + VLOG(3) << "ParallelForkJoin partition " << i << " done."; + }); + } + + // Call first compute function inline. + function(result_ptr, run_options_ptr, params, temps, &partitions[0], + prof_counters); + VLOG(3) << "ParallelForkJoin partition 0 done."; + bc.Wait(); + VLOG(2) << "ParallelForkJoin EXIT"; +} diff --git a/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h new file mode 100644 index 0000000000000000000000000000000000000000..fcf1cc62078d3847435a2e75e3ca9d109cf8b200 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/runtime_fork_join.h @@ -0,0 +1,33 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_ +#define THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_ + +#include "tensorflow/core/platform/types.h" + +extern "C" { + +// Dispatches 'num_partitions' parallel calls to 'function_ptr' and joins +// threads before returning. See comments in runtime_fork_join.cc for details. +extern void __xla_cpu_runtime_ParallelForkJoin( + void* result_ptr, const void* run_options_ptr, const void** params, + void** temps, tensorflow::uint64* prof_counters, + tensorflow::int32 num_partitions, tensorflow::int64* partitions, + tensorflow::int32 num_partitioned_dims, void* function_ptr); + +} // extern "C" + +#endif // THIRD_PARTY_TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_FORK_JOIN_H_ diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc index c614e334a8ad788e64ed38231c57925ba8b19029..fdf02e5b422f75e256feec77470bb0d079e8ef1f 100644 --- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc +++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc @@ -31,7 +31,9 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_avx.h" #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_neon.h" #include "tensorflow/compiler/xla/service/cpu/cpu_runtime_sse4_1.h" +#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h" #include "tensorflow/compiler/xla/service/cpu/runtime_conv2d.h" +#include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h" #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h" #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h" #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h" @@ -42,80 +44,6 @@ namespace xla { namespace cpu { namespace { -// Converts a symbol 'name' into the form expected by dlsym(). -std::string CanonicalizeSymbol(const std::string& name) { -#if defined(__APPLE__) - // On Mac OS X, dlsym() expects names not to be prefixed with a leading - // underscore. - if (!name.empty() && name.front() == '_') { - return name.substr(1); - } -#endif - return name; -} - -class JITSymbolTable { - public: - JITSymbolTable() { Populate(); } - - void* Lookup(llvm::StringRef jit_symbol_name) const { - auto it = jit_symbol_table_.find(jit_symbol_name); - return it == jit_symbol_table_.end() ? nullptr : it->getValue(); - } - - static bool MustBeInTable(llvm::StringRef name) { - // In particular, names starting with - // runtime::kXlaCpuRuntimeSymbolNamePrefix should not be dlsym'ed. - return name.startswith(runtime::kXlaCpuRuntimeSymbolNamePrefix); - } - - private: - void AddJITSymbolToTable(llvm::StringRef jit_symbol_name, - llvm::StringRef cpp_symbol_name, - void* jit_symbol_value) { - // The JIT symbol name and the C++ symbol name (with an extern "C" linkage) - // need to match, otherwise AOT links will fail. - CHECK(jit_symbol_name == cpp_symbol_name); - CHECK(jit_symbol_table_.insert({jit_symbol_name, jit_symbol_value}).second); - } - - void Populate() { -#define ADD_JIT_SYMBOL_TO_TABLE(base_name) \ - do { \ - AddJITSymbolToTable( \ - xla::cpu::runtime::k##base_name##SymbolName, \ - "__xla_cpu_runtime_" #base_name, \ - reinterpret_cast(__xla_cpu_runtime_##base_name)); \ - } while (false) - - ADD_JIT_SYMBOL_TO_TABLE(AcquireInfeedBufferForDequeue); - ADD_JIT_SYMBOL_TO_TABLE(ReleaseInfeedBufferAfterDequeue); - ADD_JIT_SYMBOL_TO_TABLE(AcquireOutfeedBufferForPopulation); - ADD_JIT_SYMBOL_TO_TABLE(ReleaseOutfeedBufferAfterPopulation); - ADD_JIT_SYMBOL_TO_TABLE(ExpV8F32AVX); - ADD_JIT_SYMBOL_TO_TABLE(LogV8F32AVX); - ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32SSE); - ADD_JIT_SYMBOL_TO_TABLE(LogV4F32SSE); - ADD_JIT_SYMBOL_TO_TABLE(ExpV4F32NEON); - ADD_JIT_SYMBOL_TO_TABLE(LogV4F32NEON); - ADD_JIT_SYMBOL_TO_TABLE(EigenConvF32); - ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF32); - ADD_JIT_SYMBOL_TO_TABLE(EigenMatMulF64); - ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedConvF32); - ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF32); - ADD_JIT_SYMBOL_TO_TABLE(EigenSingleThreadedMatMulF64); - -#undef ADD_JIT_SYMBOL_TO_TABLE - } - - llvm::StringMap jit_symbol_table_; -}; - -const JITSymbolTable& GetJITSymbolTable() { - static JITSymbolTable* symbol_table = new JITSymbolTable; - return *symbol_table; -} - // A simple SymbolResolver that delegates to the host dynamic linker. class SimpleResolver : public llvm::JITSymbolResolver { public: @@ -123,7 +51,6 @@ class SimpleResolver : public llvm::JITSymbolResolver { : external_constant_pool_(external_constant_pool) {} llvm::JITSymbol findSymbol(const std::string& name) override { - string name_as_string(name); if (const uint8* from_constant_pool = external_constant_pool_->Find(string(name))) { return llvm::JITEvaluatedSymbol( @@ -131,13 +58,7 @@ class SimpleResolver : public llvm::JITSymbolResolver { llvm::JITSymbolFlags::None); } - std::string canonical_name = CanonicalizeSymbol(name); - const JITSymbolTable& jit_symbol_table = GetJITSymbolTable(); - - void* func_addr = JITSymbolTable::MustBeInTable(canonical_name) - ? jit_symbol_table.Lookup(canonical_name) - : dlsym(RTLD_DEFAULT, canonical_name.c_str()); - + void* func_addr = CustomCallTargetRegistry::Global()->Lookup(name); if (func_addr == nullptr) { return nullptr; } @@ -253,5 +174,118 @@ llvm::JITSymbol SimpleOrcJIT::FindSymbol(const std::string& name) { return nullptr; } +namespace { +// Register some known symbols with the CustomCallTargetRegistry. +bool RegisterKnownJITSymbols() { + CustomCallTargetRegistry* registry = CustomCallTargetRegistry::Global(); + +#define REGISTER_CPU_RUNTIME_SYMBOL(base_name) \ + do { \ + auto* function_address = \ + reinterpret_cast(__xla_cpu_runtime_##base_name); \ + registry->Register(xla::cpu::runtime::k##base_name##SymbolName, \ + function_address); \ + CHECK_EQ( \ + tensorflow::StringPiece(xla::cpu::runtime::k##base_name##SymbolName), \ + "__xla_cpu_runtime_" #base_name); \ + } while (false) + + REGISTER_CPU_RUNTIME_SYMBOL(AcquireInfeedBufferForDequeue); + REGISTER_CPU_RUNTIME_SYMBOL(AcquireOutfeedBufferForPopulation); + REGISTER_CPU_RUNTIME_SYMBOL(EigenConvF32); + REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32); + REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64); + REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32); + REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF32); + REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF64); + REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32NEON); + REGISTER_CPU_RUNTIME_SYMBOL(ExpV4F32SSE); + REGISTER_CPU_RUNTIME_SYMBOL(ExpV8F32AVX); + REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32NEON); + REGISTER_CPU_RUNTIME_SYMBOL(LogV4F32SSE); + REGISTER_CPU_RUNTIME_SYMBOL(LogV8F32AVX); + REGISTER_CPU_RUNTIME_SYMBOL(ParallelForkJoin); + REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue); + REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation); + +#undef REGISTER_CPU_RUNTIME_SYMBOL + +#define REGISTER_LIBM_SYMBOL(name) \ + do { \ + /* Register both the F32 and F64 variants of the libm symbol. */ \ + registry->Register(#name "f", reinterpret_cast(name##f)); \ + registry->Register(#name, reinterpret_cast(name)); \ + } while (false) + + REGISTER_LIBM_SYMBOL(acos); + REGISTER_LIBM_SYMBOL(acosh); + REGISTER_LIBM_SYMBOL(asin); + REGISTER_LIBM_SYMBOL(asinh); + REGISTER_LIBM_SYMBOL(atan); + REGISTER_LIBM_SYMBOL(atan2); + REGISTER_LIBM_SYMBOL(atanh); + REGISTER_LIBM_SYMBOL(cbrt); + REGISTER_LIBM_SYMBOL(ceil); + REGISTER_LIBM_SYMBOL(copysign); + REGISTER_LIBM_SYMBOL(cos); + REGISTER_LIBM_SYMBOL(cosh); + REGISTER_LIBM_SYMBOL(erf); + REGISTER_LIBM_SYMBOL(erfc); + REGISTER_LIBM_SYMBOL(exp); + REGISTER_LIBM_SYMBOL(exp2); + REGISTER_LIBM_SYMBOL(expm1); + REGISTER_LIBM_SYMBOL(fabs); + REGISTER_LIBM_SYMBOL(fdim); + REGISTER_LIBM_SYMBOL(floor); + REGISTER_LIBM_SYMBOL(fma); + REGISTER_LIBM_SYMBOL(fmax); + REGISTER_LIBM_SYMBOL(fmin); + REGISTER_LIBM_SYMBOL(fmod); + REGISTER_LIBM_SYMBOL(frexp); + REGISTER_LIBM_SYMBOL(hypot); + REGISTER_LIBM_SYMBOL(ilogb); + REGISTER_LIBM_SYMBOL(ldexp); + REGISTER_LIBM_SYMBOL(lgamma); + REGISTER_LIBM_SYMBOL(llrint); + REGISTER_LIBM_SYMBOL(llround); + REGISTER_LIBM_SYMBOL(log); + REGISTER_LIBM_SYMBOL(log10); + REGISTER_LIBM_SYMBOL(log1p); + REGISTER_LIBM_SYMBOL(log2); + REGISTER_LIBM_SYMBOL(logb); + REGISTER_LIBM_SYMBOL(lrint); + REGISTER_LIBM_SYMBOL(lround); + REGISTER_LIBM_SYMBOL(modf); + REGISTER_LIBM_SYMBOL(nan); + REGISTER_LIBM_SYMBOL(nearbyint); + REGISTER_LIBM_SYMBOL(nextafter); + REGISTER_LIBM_SYMBOL(nexttoward); + REGISTER_LIBM_SYMBOL(pow); + REGISTER_LIBM_SYMBOL(remainder); + REGISTER_LIBM_SYMBOL(remquo); + REGISTER_LIBM_SYMBOL(rint); + REGISTER_LIBM_SYMBOL(round); + REGISTER_LIBM_SYMBOL(scalbln); + REGISTER_LIBM_SYMBOL(scalbn); + REGISTER_LIBM_SYMBOL(sin); + REGISTER_LIBM_SYMBOL(sincos); + REGISTER_LIBM_SYMBOL(sinh); + REGISTER_LIBM_SYMBOL(sqrt); + REGISTER_LIBM_SYMBOL(tan); + REGISTER_LIBM_SYMBOL(tanh); + REGISTER_LIBM_SYMBOL(tgamma); + REGISTER_LIBM_SYMBOL(trunc); + +#undef REGISTER_LIBM_SYMBOL + + registry->Register("memcpy", reinterpret_cast(memcpy)); + registry->Register("memmove", reinterpret_cast(memmove)); + registry->Register("memset", reinterpret_cast(memset)); + return true; +} + +bool unused = RegisterKnownJITSymbols(); +} // namespace + } // namespace cpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/defuser.cc b/tensorflow/compiler/xla/service/defuser.cc new file mode 100644 index 0000000000000000000000000000000000000000..d124f74d19d83269be96ee34a6b4b2a8d00a978f --- /dev/null +++ b/tensorflow/compiler/xla/service/defuser.cc @@ -0,0 +1,115 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/defuser.h" + +#include +#include +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/service/call_graph.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +namespace { + +// Copy all the instructions in the given fusion instruction into the fusion +// instruction's parent computation and replace the use of the fusion +// instruction with the copy of the fusion expression root. +Status Defuse(HloInstruction* fusion_instruction) { + VLOG(2) << "Defusing instruction: " << fusion_instruction->ToString(); + + HloComputation* fused_computation = + fusion_instruction->fused_instructions_computation(); + + // A map from fused instruction to its defused clone. + tensorflow::gtl::FlatMap + defused_instructions; + // Initialize map to contain the fusion instruction parameters mapping + // to the operands of the fusion instruction. + for (int64 i = 0; i < fusion_instruction->operand_count(); ++i) { + defused_instructions[fused_computation->parameter_instruction(i)] = + fusion_instruction->mutable_operand(i); + } + + // Create a clone of each instruction of the fused computation in the same + // computation as the fusion instruction itself. + // TODO(b/68227302): Moving instruction to new computation rather than + // cloning and deleting. + for (HloInstruction* fused_instruction : + fused_computation->MakeInstructionPostOrder()) { + if (fused_instruction->opcode() == HloOpcode::kParameter) { + continue; + } + std::vector new_operands; + for (HloInstruction* operand : fused_instruction->operands()) { + new_operands.push_back(defused_instructions.at(operand)); + } + HloInstruction* defused_instruction = + fusion_instruction->parent()->AddInstruction( + fused_instruction->CloneWithNewOperands(fused_instruction->shape(), + new_operands)); + defused_instructions[fused_instruction] = defused_instruction; + } + + TF_RETURN_IF_ERROR(fusion_instruction->ReplaceAllUsesWith( + defused_instructions.at(fusion_instruction->fused_expression_root()))); + + HloModule* module = fusion_instruction->parent()->parent(); + TF_RETURN_IF_ERROR( + fusion_instruction->parent()->RemoveInstruction(fusion_instruction)); + return module->RemoveEmbeddedComputation(fused_computation); +} + +} // namespace + +StatusOr Defuser::Run(HloModule* module) { + VLOG(1) << "Defusing module " << module->name(); + XLA_VLOG_LINES(2, "Before defusion:\n" + module->ToString()); + + bool changed = false; + std::unique_ptr call_graph = CallGraph::Build(module); + TF_RETURN_IF_ERROR(call_graph->VisitNodes( + [&](const CallGraphNode& call_graph_node) -> Status { + if (call_graph_node.computation()->IsFusionComputation()) { + TF_RET_CHECK(call_graph_node.caller_callsites().size() == 1); + HloInstruction* fusion_instruction = + call_graph_node.caller_callsites()[0].instruction(); + TF_RETURN_IF_ERROR(Defuse(fusion_instruction)); + changed = true; + } + return Status::OK(); + }, + /*visit_unreachable_nodes=*/true)); + + XLA_VLOG_LINES(2, "After defusion:\n" + module->ToString()); + + return changed; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/defuser.h b/tensorflow/compiler/xla/service/defuser.h new file mode 100644 index 0000000000000000000000000000000000000000..56b28fd22da1ea6bc19f98e76f0f2ef4044cd3af --- /dev/null +++ b/tensorflow/compiler/xla/service/defuser.h @@ -0,0 +1,41 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_ + +#include + +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" + +namespace xla { + +// A pass which replaces all fusion instructions with the equivalent un-fused +// instructions. +class Defuser : public HloPassInterface { + public: + Defuser() {} + ~Defuser() override {} + tensorflow::StringPiece name() const override { return "defuser"; } + + // Run defusion on the given module. Returns whether the module was + // changed. + StatusOr Run(HloModule* module) override; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_DEFUSER_H_ diff --git a/tensorflow/compiler/xla/service/defuser_test.cc b/tensorflow/compiler/xla/service/defuser_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..32b5c5d35fae61ae6cb17fafcada1abd6c3c088c --- /dev/null +++ b/tensorflow/compiler/xla/service/defuser_test.cc @@ -0,0 +1,214 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/defuser.h" + +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h" + +namespace op = xla::testing::opcode_matchers; + +namespace xla { +namespace { + +class DefuserTest : public HloVerifiedTestBase { + protected: + // Returns the number of fusion instructions in the module. + int FusionCount() { + int count = 0; + for (HloComputation* computation : module().computations()) { + if (computation->IsFusionComputation()) { + count++; + } + } + return count; + } + + Defuser defuser_; + const Shape shape_ = ShapeUtil::MakeShape(F32, {2, 2}); +}; + +TEST_F(DefuserTest, NoFusionInstruction) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + + module().AddEntryComputation(builder.Build()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_FALSE(defuser_.Run(&module()).ValueOrDie()); +} + +TEST_F(DefuserTest, TrivialFusionInstructionAsRoot) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction({add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(1, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Add(op::Parameter(), op::Parameter())); +} + +TEST_F(DefuserTest, TrivialFusionInstructionNotAsRoot) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction({add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Negate(op::Fusion())); + + EXPECT_EQ(1, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Negate(op::Add(op::Parameter(), op::Parameter()))); +} + +TEST_F(DefuserTest, NonTrivialFusionInstruction) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto param3 = + builder.AddInstruction(HloInstruction::CreateParameter(2, shape_, "p2")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + auto negate = builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + auto sub = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kSubtract, add, negate)); + auto mul = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kMultiply, sub, param3)); + auto div = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kDivide, mul, param3)); + auto constant = builder.AddInstruction(HloInstruction::CreateConstant( + Literal::CreateR2({{1.0, 2.0}, {3.0, 4.0}}))); + auto add2 = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, constant, div)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction( + {add2, constant, div, mul, sub, negate, add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(1, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Add(op::Constant(), op::Divide())); +} + +TEST_F(DefuserTest, MultipleFusionInstructions) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto param3 = + builder.AddInstruction(HloInstruction::CreateParameter(2, shape_, "p2")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + auto negate = builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + auto sub = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kSubtract, add, negate)); + auto mul = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kMultiply, sub, param3)); + auto div = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kDivide, mul, param3)); + auto constant = builder.AddInstruction(HloInstruction::CreateConstant( + Literal::CreateR2({{1.0, 2.0}, {3.0, 4.0}}))); + auto add2 = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, constant, div)); + + auto computation = module().AddEntryComputation(builder.Build()); + computation->CreateFusionInstruction({add2, constant, div, mul}, + HloInstruction::FusionKind::kLoop); + computation->CreateFusionInstruction({sub, negate, add}, + HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(2, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), + op::Add(op::Constant(), op::Divide())); +} + +TEST_F(DefuserTest, NestedFusionInstructions) { + auto builder = HloComputation::Builder(TestName()); + auto param0 = + builder.AddInstruction(HloInstruction::CreateParameter(0, shape_, "p0")); + auto param1 = + builder.AddInstruction(HloInstruction::CreateParameter(1, shape_, "p1")); + auto add = builder.AddInstruction( + HloInstruction::CreateBinary(shape_, HloOpcode::kAdd, param0, param1)); + auto negate = builder.AddInstruction( + HloInstruction::CreateUnary(shape_, HloOpcode::kNegate, add)); + + auto computation = module().AddEntryComputation(builder.Build()); + auto outer_fusion = computation->CreateFusionInstruction( + {negate, add}, HloInstruction::FusionKind::kLoop); + HloInstruction* fused_negate = outer_fusion->fused_expression_root(); + ASSERT_EQ(fused_negate->opcode(), HloOpcode::kNegate); + outer_fusion->fused_instructions_computation()->CreateFusionInstruction( + {fused_negate}, HloInstruction::FusionKind::kLoop); + + EXPECT_THAT(computation->root_instruction(), op::Fusion()); + + EXPECT_EQ(2, FusionCount()); + EXPECT_TRUE(defuser_.Run(&module()).ValueOrDie()); + EXPECT_EQ(0, FusionCount()); + + EXPECT_THAT(computation->root_instruction(), op::Negate(op::Add())); +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h index 5b1dbf439c7d3b02625e9d846a068b2262ceeeed..237cd8c31de1ba1aa97739c579d6d92264ddc61b 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor.h @@ -52,9 +52,6 @@ class HloInstruction; // "unimplemented" error status. // // Note: this may change to an iterator in the future for flexibility purposes. -// -// TODO(b/26548304): Stop passing in information about the visited -// instruction that is accessible from the instruction object itself. class DfsHloVisitor { public: DfsHloVisitor() {} @@ -65,120 +62,110 @@ class DfsHloVisitor { virtual Status HandleElementwiseUnary(HloInstruction* hlo); virtual Status HandleElementwiseBinary(HloInstruction* hlo); - virtual Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) = 0; - virtual Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) = 0; + virtual Status HandleClamp(HloInstruction* clamp) = 0; + virtual Status HandleSelect(HloInstruction* select) = 0; virtual Status HandleMaximum(HloInstruction* maximum) { return HandleElementwiseBinary(maximum); } virtual Status HandleMinimum(HloInstruction* minimum) { return HandleElementwiseBinary(minimum); } - virtual Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) = 0; + virtual Status HandleConcatenate(HloInstruction* concatenate) = 0; virtual Status HandleConvert(HloInstruction* convert) { return HandleElementwiseUnary(convert); } virtual Status HandleCopy(HloInstruction* copy) { return HandleElementwiseUnary(copy); } - virtual Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleComplex(HloInstruction* complex) { + return HandleElementwiseBinary(complex); + } + virtual Status HandleMultiply(HloInstruction* multiply) { return HandleElementwiseBinary(multiply); } - virtual Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) = 0; - virtual Status HandlePower(HloInstruction* power, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleDot(HloInstruction* dot) = 0; + virtual Status HandlePower(HloInstruction* power) { return HandleElementwiseBinary(power); } - virtual Status HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs, HloInstruction* rhs, - const Window& window) = 0; + virtual Status HandleConvolution(HloInstruction* convolution) = 0; virtual Status HandleCrossReplicaSum(HloInstruction* crs) = 0; - virtual Status HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) { + virtual Status HandleCompare(HloInstruction* compare) { return HandleElementwiseBinary(compare); } - virtual Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleAdd(HloInstruction* add) { return HandleElementwiseBinary(add); } - virtual Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleDivide(HloInstruction* divide) { return HandleElementwiseBinary(divide); } - virtual Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleRemainder(HloInstruction* remainder) { return HandleElementwiseBinary(remainder); } - virtual Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleSubtract(HloInstruction* subtract) { return HandleElementwiseBinary(subtract); } - virtual Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { + virtual Status HandleAbs(HloInstruction* abs) { return HandleElementwiseUnary(abs); } + virtual Status HandleAtan2(HloInstruction* atan2) { + return HandleElementwiseBinary(atan2); + } virtual Status HandleRound(HloInstruction* round) { return HandleElementwiseUnary(round); } - virtual Status HandleSign(HloInstruction* sign, HloInstruction* operand) { + virtual Status HandleSign(HloInstruction* sign) { return HandleElementwiseUnary(sign); } - virtual Status HandleNegate(HloInstruction* negate, HloInstruction* operand) { + virtual Status HandleNegate(HloInstruction* negate) { return HandleElementwiseUnary(negate); } - virtual Status HandleExp(HloInstruction* exp, HloInstruction* operand) { + virtual Status HandleExp(HloInstruction* exp) { return HandleElementwiseUnary(exp); } - virtual Status HandleFloor(HloInstruction* floor, HloInstruction* operand) { + virtual Status HandleFloor(HloInstruction* floor) { return HandleElementwiseUnary(floor); } - virtual Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) { + virtual Status HandleCeil(HloInstruction* ceil) { return HandleElementwiseUnary(ceil); } - virtual Status HandleLog(HloInstruction* log, HloInstruction* operand) { + virtual Status HandleLog(HloInstruction* log) { return HandleElementwiseUnary(log); } - virtual Status HandleCos(HloInstruction* cos, HloInstruction* operand) { + virtual Status HandleCos(HloInstruction* cos) { return HandleElementwiseUnary(cos); } - virtual Status HandleSin(HloInstruction* sin, HloInstruction* operand) { + virtual Status HandleSin(HloInstruction* sin) { return HandleElementwiseUnary(sin); } - virtual Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) { + virtual Status HandleTanh(HloInstruction* tanh) { return HandleElementwiseUnary(tanh); } - virtual Status HandleIsFinite(HloInstruction* is_finite, - HloInstruction* operand) { + virtual Status HandleReal(HloInstruction* real) { + return HandleElementwiseUnary(real); + } + virtual Status HandleImag(HloInstruction* imag) { + return HandleElementwiseUnary(imag); + } + virtual Status HandleIsFinite(HloInstruction* is_finite) { return HandleElementwiseUnary(is_finite); } - virtual Status HandleAnd(HloInstruction* and_, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleAnd(HloInstruction* and_) { return HandleElementwiseBinary(and_); } - virtual Status HandleNot(HloInstruction* not_, HloInstruction* operand) { + virtual Status HandleNot(HloInstruction* not_) { return HandleElementwiseUnary(not_); } - virtual Status HandleOr(HloInstruction* or_, HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleOr(HloInstruction* or_) { return HandleElementwiseBinary(or_); } - virtual Status HandleShiftLeft(HloInstruction* shift_left, - HloInstruction* lhs, HloInstruction* rhs) { + virtual Status HandleShiftLeft(HloInstruction* shift_left) { return HandleElementwiseBinary(shift_left); } virtual Status HandleShiftRightArithmetic( - HloInstruction* shift_right_arithmetic, HloInstruction* lhs, - HloInstruction* rhs) { + HloInstruction* shift_right_arithmetic) { return HandleElementwiseBinary(shift_right_arithmetic); } - virtual Status HandleShiftRightLogical(HloInstruction* shift_right_logical, - HloInstruction* lhs, - HloInstruction* rhs) { + virtual Status HandleShiftRightLogical(HloInstruction* shift_right_logical) { return HandleElementwiseBinary(shift_right_logical); } @@ -188,19 +175,12 @@ class DfsHloVisitor { virtual Status HandleInfeed(HloInstruction* infeed) = 0; virtual Status HandleOutfeed(HloInstruction* outfeed) = 0; - virtual Status HandleRng(HloInstruction* random, - RandomDistribution distribution) = 0; - virtual Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) = 0; - virtual Status HandleSort(HloInstruction* sort, HloInstruction* operand) = 0; - virtual Status HandleConstant(HloInstruction* constant, - const Literal& literal) = 0; - virtual Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) = 0; - virtual Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) = 0; + virtual Status HandleRng(HloInstruction* random) = 0; + virtual Status HandleReverse(HloInstruction* reverse) = 0; + virtual Status HandleSort(HloInstruction* sort) = 0; + virtual Status HandleConstant(HloInstruction* constant) = 0; + virtual Status HandleGetTupleElement(HloInstruction* get_tuple_element) = 0; + virtual Status HandleReduce(HloInstruction* reduce) = 0; virtual Status HandleBitcast(HloInstruction* bitcast) = 0; virtual Status HandleBroadcast(HloInstruction* broadcast) = 0; virtual Status HandleReshape(HloInstruction* reshape) = 0; @@ -208,31 +188,14 @@ class DfsHloVisitor { virtual Status HandleParameter(HloInstruction* parameter) = 0; virtual Status HandleFusion(HloInstruction* fusion) = 0; virtual Status HandleCall(HloInstruction* call) = 0; - virtual Status HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) = 0; - virtual Status HandleSlice(HloInstruction* slice, - HloInstruction* operand) = 0; - virtual Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) = 0; - virtual Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) = 0; - virtual Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) = 0; - virtual Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) = 0; - virtual Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, - const Window& window, - HloComputation* function) = 0; + virtual Status HandleCustomCall(HloInstruction* custom_call) = 0; + virtual Status HandleSlice(HloInstruction* slice) = 0; + virtual Status HandleDynamicSlice(HloInstruction* dynamic_slice) = 0; + virtual Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) = 0; + virtual Status HandleTuple(HloInstruction* tuple) = 0; + virtual Status HandleMap(HloInstruction* map) = 0; + virtual Status HandleReduceWindow(HloInstruction* reduce_window) = 0; virtual Status HandleSelectAndScatter(HloInstruction* instruction) = 0; virtual Status HandleWhile(HloInstruction* xla_while) = 0; diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h index a5fe120598416235dff2af9d8a5c0ae64ac9edcc..a1d7acf90429e3611bb6dea56d98bbd6ffb8f580 100644 --- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h +++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h @@ -60,14 +60,10 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { return DefaultAction(hlo); } - Status HandleClamp(HloInstruction* clamp, HloInstruction* /*min*/, - HloInstruction* /*arg*/, - HloInstruction* /*max*/) override { + Status HandleClamp(HloInstruction* clamp) override { return DefaultAction(clamp); } - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice /*operands*/) override { + Status HandleConcatenate(HloInstruction* concatenate) override { return DefaultAction(concatenate); } Status HandleConvert(HloInstruction* convert) override { @@ -76,30 +72,20 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { Status HandleCopy(HloInstruction* copy) override { return DefaultAction(copy); } - Status HandleSelect(HloInstruction* select, HloInstruction* /*pred*/, - HloInstruction* /*on_true*/, - HloInstruction* /*on_false*/) override { + Status HandleSelect(HloInstruction* select) override { return DefaultAction(select); } - Status HandleDot(HloInstruction* dot, HloInstruction* /*lhs*/, - HloInstruction* /*rhs*/) override { - return DefaultAction(dot); - } - Status HandleConvolution(HloInstruction* convolution, HloInstruction* /*lhs*/, - HloInstruction* /*rhs*/, - const Window& /*window*/) override { + Status HandleDot(HloInstruction* dot) override { return DefaultAction(dot); } + Status HandleConvolution(HloInstruction* convolution) override { return DefaultAction(convolution); } Status HandleCrossReplicaSum(HloInstruction* crs) override { return DefaultAction(crs); } - Status HandleCompare(HloInstruction* compare, HloOpcode /*opcode*/, - HloInstruction* /*lhs*/, - HloInstruction* /*rhs*/) override { + Status HandleCompare(HloInstruction* compare) override { return DefaultAction(compare); } - Status HandleRng(HloInstruction* random, - RandomDistribution /*distribution*/) override { + Status HandleRng(HloInstruction* random) override { return DefaultAction(random); } Status HandleInfeed(HloInstruction* infeed) override { @@ -108,20 +94,16 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { Status HandleOutfeed(HloInstruction* outfeed) override { return DefaultAction(outfeed); } - Status HandleReverse(HloInstruction* reverse, - HloInstruction* /*operand*/) override { + Status HandleReverse(HloInstruction* reverse) override { return DefaultAction(reverse); } - Status HandleSort(HloInstruction* sort, - HloInstruction* /*operand*/) override { + Status HandleSort(HloInstruction* sort) override { return DefaultAction(sort); } - Status HandleConstant(HloInstruction* constant, - const Literal& /*literal*/) override { + Status HandleConstant(HloInstruction* constant) override { return DefaultAction(constant); } - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* /*operand*/) override { + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override { return DefaultAction(get_tuple_element); } Status HandleParameter(HloInstruction* parameter) override { @@ -133,50 +115,27 @@ class DfsHloVisitorWithDefault : public DfsHloVisitor { Status HandleCall(HloInstruction* call) override { return DefaultAction(call); } - Status HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice /*operands*/, - tensorflow::StringPiece /*custom_call_target*/) override { + Status HandleCustomCall(HloInstruction* custom_call) override { return DefaultAction(custom_call); } - Status HandleSlice(HloInstruction* slice, - HloInstruction* /*operand*/) override { + Status HandleSlice(HloInstruction* slice) override { return DefaultAction(slice); } - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* /*operand*/, - HloInstruction* /*start_indices*/) override { + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override { return DefaultAction(dynamic_slice); } - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* /*operand*/, - HloInstruction* /*update*/, - HloInstruction* /*start_indices*/) override { + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override { return DefaultAction(dynamic_update_slice); } - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice /*operands*/) override { + Status HandleTuple(HloInstruction* tuple) override { return DefaultAction(tuple); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice /*operands*/, - HloComputation* /*function*/, - tensorflow::gtl::ArraySlice /*static_operands*/) - override { - return DefaultAction(map); - } - Status HandleReduce(HloInstruction* reduce, HloInstruction* /*arg*/, - HloInstruction* /*init_value*/, - tensorflow::gtl::ArraySlice /*dimensions*/, - HloComputation* /*function*/) override { + Status HandleMap(HloInstruction* map) override { return DefaultAction(map); } + Status HandleReduce(HloInstruction* reduce) override { return DefaultAction(reduce); } - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* /*operand*/, - const Window& /*window*/, - HloComputation* /*function*/) override { + Status HandleReduceWindow(HloInstruction* reduce_window) override { return DefaultAction(reduce_window); } Status HandleSelectAndScatter(HloInstruction* select_and_scatter) override { diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc index 44f709bedec7ef0e50b830e8901796985ee7224e..fd4c332cba94513ec5b4cd88a842189e716f35d5 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.cc @@ -54,10 +54,12 @@ StatusOr ElementalIrEmitter::EmitUnaryOp( const HloInstruction* op, llvm::Value* operand_value) const { if (op->opcode() == HloOpcode::kCopy) { return operand_value; + } else if (operand_value->getType()->isIntegerTy()) { + return EmitIntegerUnaryOp(op, operand_value); + } else if (ShapeUtil::ElementIsComplex(op->operand(0)->shape())) { + return EmitComplexUnaryOp(op, operand_value); } else { - return operand_value->getType()->isIntegerTy() - ? EmitIntegerUnaryOp(op, operand_value) - : EmitFloatUnaryOp(op, operand_value); + return EmitFloatUnaryOp(op, operand_value); } } @@ -73,20 +75,35 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( } if (primitive_util::IsIntegralType(to_type)) { return ir_builder_->CreateIntCast( - operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_), + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_), primitive_util::IsSignedIntegralType(to_type)); } if (primitive_util::IsFloatingPointType(to_type)) { if (primitive_util::IsSignedIntegralType(from_type)) { return ir_builder_->CreateSIToFP( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } if (primitive_util::IsUnsignedIntegralType(from_type) || from_type == PRED) { return ir_builder_->CreateUIToFP( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); + } + } + if (primitive_util::IsComplexType(to_type)) { + auto to_ir_component_type = llvm_ir::PrimitiveTypeToIrType( + primitive_util::ComplexComponentType(to_type), module_); + if (primitive_util::IsSignedIntegralType(from_type)) { + return ComposeComplex( + op, + ir_builder_->CreateSIToFP(operand_value, to_ir_component_type), + nullptr); + } + if (primitive_util::IsUnsignedIntegralType(from_type) || + from_type == PRED) { + return ComposeComplex( + op, + ir_builder_->CreateUIToFP(operand_value, to_ir_component_type), + nullptr); } } return Unimplemented("conversion from primitive type %s to %s", @@ -97,8 +114,8 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( bool is_signed = primitive_util::IsSignedIntegralType(op->shape().element_type()); if (is_signed) { - auto type = llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), - ir_builder_); + auto type = + llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_); auto zero = llvm::ConstantInt::get(type, 0); auto cmp = ir_builder_->CreateICmpSGE(operand_value, zero); return ir_builder_->CreateSelect(cmp, operand_value, @@ -110,8 +127,8 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( case HloOpcode::kSign: { bool is_signed = primitive_util::IsSignedIntegralType(op->shape().element_type()); - auto type = llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), - ir_builder_); + auto type = + llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_); auto zero = llvm::ConstantInt::get(type, 0); auto cmp = ir_builder_->CreateICmpEQ(operand_value, zero); if (is_signed) { @@ -135,7 +152,7 @@ StatusOr ElementalIrEmitter::EmitIntegerUnaryOp( return ir_builder_->CreateZExt( ir_builder_->CreateNot(ir_builder_->CreateTrunc( operand_value, ir_builder_->getInt1Ty())), - llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_)); + llvm_ir::PrimitiveTypeToIrType(PRED, module_)); } else if (primitive_util::IsIntegralType(type)) { return ir_builder_->CreateNot(operand_value); } @@ -157,20 +174,30 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( if (from_type == to_type) { return operand_value; } + if (primitive_util::IsComplexType(to_type)) { + PrimitiveType to_component_type = + primitive_util::ComplexComponentType(to_type); + if (from_type == to_component_type) { + return ComposeComplex(op, operand_value, nullptr); + } + return ComposeComplex( + op, + ir_builder_->CreateFPCast( + operand_value, + llvm_ir::PrimitiveTypeToIrType(to_component_type, module_)), + nullptr); + } if (primitive_util::IsFloatingPointType(to_type)) { return ir_builder_->CreateFPCast( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } if (primitive_util::IsSignedIntegralType(to_type)) { return ir_builder_->CreateFPToSI( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } if (primitive_util::IsUnsignedIntegralType(to_type)) { return ir_builder_->CreateFPToUI( - operand_value, - llvm_ir::PrimitiveTypeToIrType(to_type, ir_builder_)); + operand_value, llvm_ir::PrimitiveTypeToIrType(to_type, module_)); } return Unimplemented("unhandled conversion operation: %s => %s", PrimitiveType_Name(from_type).c_str(), @@ -230,7 +257,7 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( auto not_infinite = ir_builder_->CreateFCmpONE(abs_value, infinity); auto result_i1 = ir_builder_->CreateAnd(equal_self, not_infinite); return ir_builder_->CreateZExt( - result_i1, llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder_)); + result_i1, llvm_ir::PrimitiveTypeToIrType(PRED, module_)); } case HloOpcode::kNegate: return ir_builder_->CreateFNeg(operand_value); @@ -240,20 +267,164 @@ StatusOr ElementalIrEmitter::EmitFloatUnaryOp( } } +StatusOr ElementalIrEmitter::EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + switch (op->opcode()) { + // TODO(b/65209142): Angle/Log require atan2. + // case HloOpcode::kAngle: + // case HloOpcode::kLog: // log(a+bi) = .5*log(a^2+b^2) + i*atan2(b, a) + case HloOpcode::kConvert: { + PrimitiveType from_type = op->operand(0)->shape().element_type(); + TF_RET_CHECK(primitive_util::IsComplexType(from_type)); + PrimitiveType to_type = op->shape().element_type(); + TF_RET_CHECK(primitive_util::IsComplexType(to_type)); + if (from_type == to_type) { + return operand_value; + } + PrimitiveType to_component_type = + primitive_util::ComplexComponentType(to_type); + auto to_ir_component_type = + llvm_ir::PrimitiveTypeToIrType(to_component_type, module_); + return ComposeComplex( + op, + ir_builder_->CreateFPCast(real(operand_value), to_ir_component_type), + ir_builder_->CreateFPCast(imag(operand_value), to_ir_component_type)); + } + case HloOpcode::kExp: { + // e^(a+bi) = e^a*(cos(b)+sin(b)i) + auto exp_a = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::exp, {real(operand_value)}, + {real(operand_value)->getType()}, ir_builder_); + auto cos_b = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::cos, {imag(operand_value)}, + {imag(operand_value)->getType()}, ir_builder_); + auto sin_b = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::sin, {imag(operand_value)}, + {imag(operand_value)->getType()}, ir_builder_); + return ComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b), + ir_builder_->CreateFMul(exp_a, sin_b)); + } + case HloOpcode::kCos: { + // cos(z) = .5(e^(iz) + e^(-iz)) + // cos(a+bi) = .5(e^(-b+ai) + e^(b-ai)) + // now, e^(x+yi) = e^x*(cos(y)+sin(y)i), so we have + // cos(a+bi) = .5(e^-b*(cos(a)+sin(a)i) + e^b*(cos(-a)+sin(-a)i)) + // cos(-x) = cos(x) and sin(-x) = -sin(x), so + // cos(a+bi) = .5(e^-b*(cos(a)+sin(a)i) + e^b*(cos(a)-sin(a)i)) + // = .5(cos(a)*(e^-b+e^b) + i*sin(a)*(e^-b-e^b)) + auto a = real(operand_value); + auto b = imag(operand_value); + auto type = a->getType(); + auto exp_b = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {b}, + {type}, ir_builder_); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b); + auto cos_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {a}, + {type}, ir_builder_); + auto sin_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {a}, + {type}, ir_builder_); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)), + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b))); + } + case HloOpcode::kSin: { + // sin(z) = .5i(e^(-iz) - e^(iz)) + // sin(a+bi) = .5i(e^(-i(a+bi)) - e^(i(a+bi))) + // = .5i(e^(b-ai) - e^(-b+ai)) + // now, e^(x+yi) = e^x*(cos(y)+sin(y)i), so we have + // sin(a+bi) = 0.5i(e^b*(cos(-a)+sin(-a)i) - e^-b*(cos(a)+sin(a)i)) + // = 0.5(e^b*(cos(-a)i-sin(-a)) - e^-b*(cos(a)i-sin(a))) + // cos(-x) = cos(x) and sin(-x) = -sin(x), so + // = 0.5(e^b*(cos(a)i+sin(a)) - e^-b*(cos(a)i-sin(a))) + // = 0.5(sin(a)*(e^b+e^-b) + i*cos(a)*(e^b-e^-b) + auto a = real(operand_value); + auto b = imag(operand_value); + auto type = a->getType(); + auto exp_b = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::exp, {b}, + {type}, ir_builder_); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(type, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(type, 0.5), exp_b); + auto cos_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::cos, {a}, + {type}, ir_builder_); + auto sin_a = llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sin, {a}, + {type}, ir_builder_); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)), + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b))); + } + case HloOpcode::kAbs: { + auto sum_sq = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(operand_value), real(operand_value)), + ir_builder_->CreateFMul(imag(operand_value), imag(operand_value))); + return llvm_ir::EmitCallToIntrinsic(llvm::Intrinsic::sqrt, {sum_sq}, + {sum_sq->getType()}, ir_builder_); + } + case HloOpcode::kSign: { // Sign(c) = c / |c| + auto sum_sq = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(operand_value), real(operand_value)), + ir_builder_->CreateFMul(imag(operand_value), imag(operand_value))); + auto cplx_abs = llvm_ir::EmitCallToIntrinsic( + llvm::Intrinsic::sqrt, {sum_sq}, {sum_sq->getType()}, ir_builder_); + auto type = cplx_abs->getType(); + auto zero = llvm::ConstantFP::get(type, 0.0); + auto oeq = ir_builder_->CreateFCmpOEQ(cplx_abs, zero); + return ir_builder_->CreateSelect( + oeq, ComposeComplex(op, zero, zero), + ComposeComplex( + op, ir_builder_->CreateFDiv(real(operand_value), cplx_abs), + ir_builder_->CreateFDiv(imag(operand_value), cplx_abs))); + } + case HloOpcode::kNegate: + return ComposeComplex(op, ir_builder_->CreateFNeg(real(operand_value)), + ir_builder_->CreateFNeg(imag(operand_value))); + case HloOpcode::kReal: + return real(operand_value); + case HloOpcode::kImag: + return imag(operand_value); + default: + return Unimplemented("unary complex op '%s'", + HloOpcodeString(op->opcode()).c_str()); + } +} + StatusOr ElementalIrEmitter::EmitBinaryOp( const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const { - return lhs_value->getType()->isIntegerTy() - ? EmitIntegerBinaryOp(op, lhs_value, rhs_value, - primitive_util::IsSignedIntegralType( - op->operand(0)->shape().element_type())) - : EmitFloatBinaryOp(op, lhs_value, rhs_value); + PrimitiveType operand_type = op->operand(0)->shape().element_type(); + if (lhs_value->getType()->isIntegerTy()) { + return EmitIntegerBinaryOp( + op, lhs_value, rhs_value, + primitive_util::IsSignedIntegralType(operand_type)); + } else if (primitive_util::IsComplexType(operand_type)) { + return EmitComplexBinaryOp(op, lhs_value, rhs_value); + } else { + return EmitFloatBinaryOp(op, lhs_value, rhs_value); + } } StatusOr ElementalIrEmitter::EmitFloatBinaryOp( const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const { switch (op->opcode()) { + // case HloOpcode::kAtan2: // TODO(b/65209142): CPU atan2 support + case HloOpcode::kComplex: + return ComposeComplex(op, lhs_value, rhs_value); case HloOpcode::kAdd: return ir_builder_->CreateFAdd(lhs_value, rhs_value); case HloOpcode::kSubtract: @@ -305,6 +476,88 @@ StatusOr ElementalIrEmitter::EmitFloatBinaryOp( } } +StatusOr ElementalIrEmitter::EmitComplexBinaryOp( + const HloInstruction* op, llvm::Value* lhs_value, + llvm::Value* rhs_value) const { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + switch (op->opcode()) { + case HloOpcode::kAdd: + return ComposeComplex( + op, ir_builder_->CreateFAdd(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFAdd(imag(lhs_value), imag(rhs_value))); + case HloOpcode::kSubtract: + return ComposeComplex( + op, ir_builder_->CreateFSub(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFSub(imag(lhs_value), imag(rhs_value))); + case HloOpcode::kMultiply: + return ComposeComplex( + op, + ir_builder_->CreateFSub( + ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), imag(rhs_value))), + ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_value), imag(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value)))); + case HloOpcode::kDivide: { + // (a+bi) / (c+di) = ((a+bi)(c-di)) / ((c+di)(c-di)) + // = ((ac + bd) + (bc - ad)i) / (c^2 + d^2) + auto rhs_sum_sq = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(rhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(rhs_value), imag(rhs_value))); + auto type = rhs_sum_sq->getType(); + auto zero = llvm::ConstantFP::get(type, 0.0); + auto oeq = ir_builder_->CreateFCmpOEQ(rhs_sum_sq, zero); + return ir_builder_->CreateSelect( + oeq, ComposeComplex(op, llvm::ConstantFP::getInfinity(type), zero), + ComposeComplex( + op, + ir_builder_->CreateFDiv( + ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), + imag(rhs_value))), + rhs_sum_sq), + ir_builder_->CreateFDiv( + ir_builder_->CreateFSub( + ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(real(lhs_value), + imag(rhs_value))), + rhs_sum_sq))); + } + // LLVM comparisons can be "unordered" (U) or "ordered" (O) -- ordered + // comparisons always return false when one of the operands is NaN, whereas + // unordered comparisons return true. + // + // We use ordered comparisons for everything except kNe, where we use an + // unordered comparison. This makes x != y equivalent to !(x == y), and + // matches C++'s semantics. + case HloOpcode::kEq: + return ir_builder_->CreateAnd( + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, real(lhs_value), + real(rhs_value), ir_builder_), + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_OEQ, imag(lhs_value), + imag(rhs_value), ir_builder_)); + case HloOpcode::kNe: + return ir_builder_->CreateOr( + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, real(lhs_value), + real(rhs_value), ir_builder_), + llvm_ir::EmitComparison(llvm::CmpInst::FCMP_UNE, imag(lhs_value), + imag(rhs_value), ir_builder_)); + + // TODO(b/65209142): requires arg(z) -> requires atan|atan2 intrinsic + // case HloOpcode::kPower: + // // (a+bi)^(c+di) = exp(i(c+di)*arg(a+bi)) * (a*a+b*b)^(c/2+di/2) + default: + return Unimplemented("binary complex op '%s'", + HloOpcodeString(op->opcode()).c_str()); + } +} + llvm::Value* ElementalIrEmitter::EmitFloatMax(llvm::Value* lhs_value, llvm::Value* rhs_value) const { return llvm_ir::EmitFloatMax(lhs_value, rhs_value, ir_builder_); @@ -396,7 +649,7 @@ StatusOr ElementalIrEmitter::EmitErfInv(PrimitiveType prim_type, StatusOr ElementalIrEmitter::EmitErfcInv( PrimitiveType prim_type, llvm::Value* value) const { // Compute erfcinv(value) by calculating erfinv(1.0 - value). - auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, ir_builder_); + auto type = llvm_ir::PrimitiveTypeToIrType(prim_type, module_); auto one = llvm::ConstantFP::get(type, 1.0); return EmitErfInv(prim_type, ir_builder_->CreateFSub(one, value)); } @@ -619,7 +872,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator( const { PrimitiveType param_prim_type = hlo->operand(0)->shape().element_type(); llvm::Type* param_ir_type = - llvm_ir::PrimitiveTypeToIrType(param_prim_type, ir_builder_); + llvm_ir::PrimitiveTypeToIrType(param_prim_type, module_); // Same values as PCG library // https://github.com/imneme/pcg-c/blob/master/include/pcg_variants.h @@ -783,7 +1036,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeRngElementGenerator( return ir_builder_->CreateZExt( ir_builder_->CreateFCmpOLT(get_next_uniform_float(), p), llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_)); + module_)); } default: return InvalidArgument( @@ -806,9 +1059,11 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( case HloOpcode::kCos: case HloOpcode::kExp: case HloOpcode::kFloor: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLog: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kTanh: @@ -821,6 +1076,8 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( return EmitUnaryOp(hlo, operand_value); }; case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kEq: case HloOpcode::kGe: @@ -913,10 +1170,10 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( } llvm_ir::SetToFirstInsertPoint(exit_block, ir_builder_); - llvm::PHINode* output = ir_builder_->CreatePHI( - llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_), - hlo->operands().size()); + llvm::PHINode* output = + ir_builder_->CreatePHI(llvm_ir::PrimitiveTypeToIrType( + hlo->shape().element_type(), module_), + hlo->operands().size()); auto prior_insert_point = ir_builder_->GetInsertPoint(); ir_builder_->SetInsertPoint(init_block); @@ -1075,7 +1332,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( // else -> return data from 'index'. llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry( llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_), + module_), "ret_value_addr", ir_builder_); llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse( slice_intersection, "slice_intersection", ir_builder_); @@ -1164,7 +1421,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( // } llvm::Value* ret_value_addr = llvm_ir::EmitAllocaAtFunctionEntry( llvm_ir::PrimitiveTypeToIrType(hlo->shape().element_type(), - ir_builder_), + module_), "pad_result_addr", ir_builder_); llvm_ir::LlvmIfData if_data = llvm_ir::EmitIfThenElse(in_bounds, "in_bounds", ir_builder_); @@ -1206,7 +1463,7 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( ir_builder_); PrimitiveType primitive_type = hlo->shape().element_type(); llvm::Type* primitive_type_llvm = - llvm_ir::PrimitiveTypeToIrType(primitive_type, ir_builder_); + llvm_ir::PrimitiveTypeToIrType(primitive_type, module_); llvm::Value* accumulator_alloca = llvm_ir::EmitAllocaAtFunctionEntry( primitive_type_llvm, "dot_acc", ir_builder_); ir_builder_->CreateStore( @@ -1239,7 +1496,28 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( TF_ASSIGN_OR_RETURN(llvm::Value * lhs_value, lhs_generator(lhs_index)); TF_ASSIGN_OR_RETURN(llvm::Value * rhs_value, rhs_generator(rhs_index)); llvm::Value* next_accumulator; - if (primitive_util::IsFloatingPointType(primitive_type)) { + if (primitive_util::IsComplexType(primitive_type)) { + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + llvm::Value* product_real = ir_builder_->CreateFSub( + ir_builder_->CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), imag(rhs_value))); + llvm::Value* product_imag = ir_builder_->CreateFAdd( + ir_builder_->CreateFMul(real(lhs_value), imag(rhs_value)), + ir_builder_->CreateFMul(imag(lhs_value), real(rhs_value))); + next_accumulator = ir_builder_->CreateInsertValue( + current_accumulator, + ir_builder_->CreateFAdd(real(current_accumulator), product_real), + {0}); + next_accumulator = ir_builder_->CreateInsertValue( + next_accumulator, + ir_builder_->CreateFAdd(imag(current_accumulator), product_imag), + {1}); + } else if (primitive_util::IsFloatingPointType(primitive_type)) { next_accumulator = ir_builder_->CreateFAdd( current_accumulator, ir_builder_->CreateFMul(lhs_value, rhs_value)); @@ -1261,4 +1539,17 @@ llvm_ir::ElementGenerator ElementalIrEmitter::MakeElementGenerator( } } +llvm::Value* ElementalIrEmitter::ComposeComplex(const HloInstruction* op, + llvm::Value* real, + llvm::Value* imag) const { + auto cplx_type = + llvm_ir::PrimitiveTypeToIrType(op->shape().element_type(), module_); + auto complex = ir_builder_->CreateInsertValue( + llvm::ConstantAggregateZero::get(cplx_type), real, {0}); + if (imag != nullptr) { + complex = ir_builder_->CreateInsertValue(complex, imag, {1}); + } + return complex; +} + } // namespace xla diff --git a/tensorflow/compiler/xla/service/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/elemental_ir_emitter.h index 35dfa88e9b02e3ec7686dc7fdded8cf4e88201fb..9d32436e38fa2fb3e27d09f01b860cd2edf2c8ac 100644 --- a/tensorflow/compiler/xla/service/elemental_ir_emitter.h +++ b/tensorflow/compiler/xla/service/elemental_ir_emitter.h @@ -55,6 +55,7 @@ class ElementalIrEmitter { const HloToElementGeneratorMap& operand_to_generator) const; llvm::IRBuilder<>* ir_builder() const { return ir_builder_; } + llvm::Module* module() const { return module_; } protected: virtual StatusOr EmitIntegerUnaryOp( @@ -63,6 +64,9 @@ class ElementalIrEmitter { virtual StatusOr EmitFloatUnaryOp( const HloInstruction* op, llvm::Value* operand_value) const; + virtual StatusOr EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const; + virtual StatusOr EmitIntegerBinaryOp(const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value, @@ -72,6 +76,10 @@ class ElementalIrEmitter { const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const; + virtual StatusOr EmitComplexBinaryOp( + const HloInstruction* op, llvm::Value* lhs_value, + llvm::Value* rhs_value) const; + virtual llvm::Value* EmitFloatMax(llvm::Value* lhs_value, llvm::Value* rhs_value) const; @@ -109,6 +117,11 @@ class ElementalIrEmitter { // compiled executable outside of the HLO code itself. const HloModuleConfig& hlo_module_config_; + protected: + // Composes a complex struct. imag may be nullptr for simple cast operations. + llvm::Value* ComposeComplex(const HloInstruction* op, llvm::Value* real, + llvm::Value* imag) const; + private: // Returns a ElementGenerator for a RNG HloInstruction. llvm_ir::ElementGenerator MakeRngElementGenerator( diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc index 62b8fa6a2b77e21ae3aa257935f5a22e3e8a130b..9c96d9eb30b5f9e51b7f5d82391c6b9f366898d6 100644 --- a/tensorflow/compiler/xla/service/executable.cc +++ b/tensorflow/compiler/xla/service/executable.cc @@ -17,7 +17,9 @@ limitations under the License. #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h" #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h" +#include "tensorflow/compiler/xla/status.h" #include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/env.h" @@ -82,7 +84,11 @@ Status Executable::DumpSessionModule() { } filename = SanitizeFileName(std::move(filename)); string file_path = tensorflow::io::JoinPath(directory_path, filename); - return tensorflow::WriteBinaryProto(env, file_path, session_module); + string result; + TF_RET_CHECK( + tensorflow::SerializeToStringDeterministic(session_module, &result)); + return tensorflow::WriteStringToFile(tensorflow::Env::Default(), file_path, + result); } } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/copy_thunk.cc b/tensorflow/compiler/xla/service/gpu/copy_thunk.cc index 87858e94090d1f7506ee09b9015b4417aee55707..f4498663b1c039b3175376baf8f27c4ecec678ec 100644 --- a/tensorflow/compiler/xla/service/gpu/copy_thunk.cc +++ b/tensorflow/compiler/xla/service/gpu/copy_thunk.cc @@ -20,15 +20,16 @@ limitations under the License. namespace xla { namespace gpu { -CopyThunk::CopyThunk(const void* source_address, - const BufferAllocation::Slice& destination_buffer, - uint64 mem_size, const HloInstruction* hlo_instruction) +HostToDeviceCopyThunk::HostToDeviceCopyThunk( + const void* source_address, + const BufferAllocation::Slice& destination_buffer, uint64 mem_size, + const HloInstruction* hlo_instruction) : Thunk(Kind::kCopy, hlo_instruction), source_address_(source_address), destination_buffer_(destination_buffer), mem_size_(mem_size) {} -tensorflow::Status CopyThunk::ExecuteOnStream( +tensorflow::Status HostToDeviceCopyThunk::ExecuteOnStream( const BufferAllocations& buffer_allocations, perftools::gputools::Stream* stream) { perftools::gputools::DeviceMemoryBase destination_data = @@ -37,5 +38,24 @@ tensorflow::Status CopyThunk::ExecuteOnStream( return tensorflow::Status::OK(); } +DeviceToDeviceCopyThunk::DeviceToDeviceCopyThunk( + const BufferAllocation::Slice& source_buffer, + const BufferAllocation::Slice& destination_buffer, uint64 mem_size, + const HloInstruction* hlo_instruction) + : Thunk(Kind::kCopy, hlo_instruction), + source_buffer_(source_buffer), + destination_buffer_(destination_buffer), + mem_size_(mem_size) {} + +tensorflow::Status DeviceToDeviceCopyThunk::ExecuteOnStream( + const BufferAllocations& buffer_allocations, + perftools::gputools::Stream* stream) { + perftools::gputools::DeviceMemoryBase destination_data = + buffer_allocations.GetDeviceAddress(destination_buffer_); + perftools::gputools::DeviceMemoryBase source_data = + buffer_allocations.GetDeviceAddress(source_buffer_); + stream->ThenMemcpy(&destination_data, source_data, mem_size_); + return tensorflow::Status::OK(); +} } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/copy_thunk.h b/tensorflow/compiler/xla/service/gpu/copy_thunk.h index 6b8c432715f27fc02b13fc242db5ee6db098c47e..e2783fd255239d31edc89701ea208f33ebb8d3fb 100644 --- a/tensorflow/compiler/xla/service/gpu/copy_thunk.h +++ b/tensorflow/compiler/xla/service/gpu/copy_thunk.h @@ -26,19 +26,18 @@ limitations under the License. namespace xla { namespace gpu { -// A thunk that copies data. For now, it copies data only from host to device. -// But it can be extended to perform device-to-host or intra-device copying. -class CopyThunk : public Thunk { +// A thunk that copies data from a host buffer to a device buffer. +class HostToDeviceCopyThunk : public Thunk { public: // Constructs a CopyThunk that copies host data from `source_address` to the // device buffer `destination_buffer`. `mem_size` is the size of the data in // bytes. - CopyThunk(const void* source_address, - const BufferAllocation::Slice& destination_buffer, uint64 mem_size, - const HloInstruction* hlo_instruction); + HostToDeviceCopyThunk(const void* source_address, + const BufferAllocation::Slice& destination_buffer, + uint64 mem_size, const HloInstruction* hlo_instruction); - CopyThunk(const CopyThunk&) = delete; - CopyThunk& operator=(const CopyThunk&) = delete; + HostToDeviceCopyThunk(const HostToDeviceCopyThunk&) = delete; + HostToDeviceCopyThunk& operator=(const HostToDeviceCopyThunk&) = delete; tensorflow::Status ExecuteOnStream( const BufferAllocations& buffer_allocations, @@ -50,6 +49,30 @@ class CopyThunk : public Thunk { const uint64 mem_size_; }; +// A thunk that copies data from a device buffer to another device buffer. +class DeviceToDeviceCopyThunk : public Thunk { + public: + // Constructs a CopyThunk that copies host data from `source_buffer` to the + // device buffer `destination_buffer`. `mem_size` is the size of the data in + // bytes. + DeviceToDeviceCopyThunk(const BufferAllocation::Slice& source_buffer, + const BufferAllocation::Slice& destination_buffer, + uint64 mem_size, + const HloInstruction* hlo_instruction); + + DeviceToDeviceCopyThunk(const DeviceToDeviceCopyThunk&) = delete; + DeviceToDeviceCopyThunk& operator=(const DeviceToDeviceCopyThunk&) = delete; + + tensorflow::Status ExecuteOnStream( + const BufferAllocations& buffer_allocations, + perftools::gputools::Stream* stream) override; + + private: + const BufferAllocation::Slice source_buffer_; + const BufferAllocation::Slice destination_buffer_; + const uint64 mem_size_; +}; + } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc index 8810a85ceeafd8b2d9ad8d7412266847abe5b75d..1b94499bc6ef6d587cdb1fafec48bc4e5b917c51 100644 --- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.cc @@ -135,6 +135,10 @@ StatusOr GpuElementalIrEmitter::EmitFloatBinaryOp( PrimitiveType rhs_input_type = op->operand(1)->shape().element_type(); PrimitiveType output_type = op->shape().element_type(); switch (op->opcode()) { + case HloOpcode::kAtan2: + return EmitLibdeviceMathCall("__nv_atan2", {lhs_value, rhs_value}, + {lhs_input_type, rhs_input_type}, + output_type); case HloOpcode::kRemainder: { return EmitLibdeviceMathCall("__nv_fmod", {lhs_value, rhs_value}, {lhs_input_type, rhs_input_type}, @@ -226,6 +230,112 @@ StatusOr GpuElementalIrEmitter::EmitFloatUnaryOp( } } +StatusOr GpuElementalIrEmitter::EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const { + PrimitiveType input_type = op->operand(0)->shape().element_type(); + PrimitiveType component_type = + primitive_util::IsComplexType(input_type) + ? primitive_util::ComplexComponentType(input_type) + : input_type; + auto real = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_->CreateExtractValue(x, {1}); + }; + + switch (op->opcode()) { + case HloOpcode::kLog: { + // log(a+bi) = .5*log(a^2+b^2) + i*atan2(b, a) + auto a = real(operand_value); + auto b = imag(operand_value); + llvm::Type* llvm_ty = a->getType(); + auto sum_sq = ir_builder_->CreateFAdd(ir_builder_->CreateFMul(a, a), + ir_builder_->CreateFMul(b, b)); + TF_ASSIGN_OR_RETURN( + auto log_sum_sq, + EmitLibdeviceMathCall("__nv_log", {sum_sq}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto angle, EmitLibdeviceMathCall("__nv_atan2", {b, a}, + {component_type, component_type}, + component_type)); + auto one_half = llvm::ConstantFP::get(llvm_ty, 0.5); + return ComposeComplex(op, ir_builder_->CreateFMul(one_half, log_sum_sq), + angle); + } + // TODO(b/65408531): Implement kPower on GPU, where atan2 is available. + // case HloOpcode::kPower: + // // (a+bi)^(c+di) = exp(i(c+di)*arg(a+bi)) * (a*a+b*b)^(0.5(c+di)) + case HloOpcode::kExp: { + // e^(a+bi) = e^a*(cos(b)+sin(b)i) + auto b = imag(operand_value); + TF_ASSIGN_OR_RETURN( + auto exp_a, EmitLibdeviceMathCall("__nv_exp", {real(operand_value)}, + {component_type}, component_type)); + TF_ASSIGN_OR_RETURN( + auto cos_b, EmitLibdeviceMathCall("__nv_cos", {b}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto sin_b, EmitLibdeviceMathCall("__nv_sin", {b}, {component_type}, + component_type)); + return ComposeComplex(op, ir_builder_->CreateFMul(exp_a, cos_b), + ir_builder_->CreateFMul(exp_a, sin_b)); + } + case HloOpcode::kCos: { + // cos(a+bi) = .5(cos(a)*(e^-b+e^b) + i*sin(a)*(e^-b-e^b)) + auto a = real(operand_value); + auto llvm_ty = a->getType(); + TF_ASSIGN_OR_RETURN( + auto exp_b, EmitLibdeviceMathCall("__nv_exp", {imag(operand_value)}, + {component_type}, component_type)); + TF_ASSIGN_OR_RETURN( + auto cos_a, EmitLibdeviceMathCall("__nv_cos", {a}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto sin_a, EmitLibdeviceMathCall("__nv_sin", {a}, {component_type}, + component_type)); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFAdd(half_exp_neg_b, half_exp_b)), + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFSub(half_exp_neg_b, half_exp_b))); + } + + case HloOpcode::kSin: { + // sin(a+bi) = 0.5(sin(a)*(e^b+e^-b) + i*cos(a)*(e^b-e^-b) + auto a = real(operand_value); + auto llvm_ty = a->getType(); + TF_ASSIGN_OR_RETURN( + auto exp_b, EmitLibdeviceMathCall("__nv_exp", {imag(operand_value)}, + {component_type}, component_type)); + TF_ASSIGN_OR_RETURN( + auto cos_a, EmitLibdeviceMathCall("__nv_cos", {a}, {component_type}, + component_type)); + TF_ASSIGN_OR_RETURN( + auto sin_a, EmitLibdeviceMathCall("__nv_sin", {a}, {component_type}, + component_type)); + auto half_exp_b = + ir_builder_->CreateFMul(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + auto half_exp_neg_b = + ir_builder_->CreateFDiv(llvm::ConstantFP::get(llvm_ty, 0.5), exp_b); + return ComposeComplex( + op, + ir_builder_->CreateFMul( + sin_a, ir_builder_->CreateFAdd(half_exp_b, half_exp_neg_b)), + ir_builder_->CreateFMul( + cos_a, ir_builder_->CreateFSub(half_exp_b, half_exp_neg_b))); + } + default: + return ElementalIrEmitter::EmitComplexUnaryOp(op, operand_value); + } +} + llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall( const string& callee_name, tensorflow::gtl::ArraySlice operands, @@ -235,13 +345,12 @@ llvm::Value* GpuElementalIrEmitter::EmitDeviceFunctionCall( std::vector ir_input_types; for (PrimitiveType input_type : input_types) { ir_input_types.push_back( - llvm_ir::PrimitiveTypeToIrType(input_type, ir_builder_)); + llvm_ir::PrimitiveTypeToIrType(input_type, module_)); } llvm::FunctionType* callee_type = llvm::FunctionType::get( - llvm_ir::PrimitiveTypeToIrType(output_type, - ir_builder_), // The return type. - ir_input_types, // The parameter types. - false); // No variadic arguments. + llvm_ir::PrimitiveTypeToIrType(output_type, module_), // Return type. + ir_input_types, // Parameter types. + false); // No variadic arguments. // Declares the callee if it is not declared already. llvm::Function* callee = llvm::cast( @@ -315,7 +424,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator( PrimitiveType operand_element_type = operand->shape().element_type(); llvm::Value* accum_ptr = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, module_), "reduce_window_accum_ptr", ir_builder_); { TF_ASSIGN_OR_RETURN(llvm::Value * init_value, @@ -377,7 +486,7 @@ llvm_ir::ElementGenerator GpuElementalIrEmitter::MakeElementGenerator( const HloInstruction* operand = hlo->operand(0); llvm::Value* accum_ptr = ir_builder()->CreateAlloca(llvm_ir::PrimitiveTypeToIrType( - hlo->shape().element_type(), ir_builder())); + hlo->shape().element_type(), module_)); TF_ASSIGN_OR_RETURN(llvm::Value * init_value, operand_to_generator.at(hlo->operand(1))({})); ir_builder()->CreateStore(init_value, accum_ptr); diff --git a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h index 6ddfc3710c56a4e129f050f862812a3d78d8dba0..3defa1b696d3addc012702e23102bb1fa140170d 100644 --- a/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/elemental_ir_emitter.h @@ -54,6 +54,9 @@ class GpuElementalIrEmitter : public ElementalIrEmitter { StatusOr EmitFloatUnaryOp( const HloInstruction* op, llvm::Value* operand_value) const override; + StatusOr EmitComplexUnaryOp( + const HloInstruction* op, llvm::Value* operand_value) const override; + StatusOr EmitFloatBinaryOp( const HloInstruction* op, llvm::Value* lhs_value, llvm::Value* rhs_value) const override; diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 57f11db11f1e207408e490398ce5038ae45dd6de..b5331fe4e2ba34443555e9bf46dfc188cbd6548a 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -67,6 +67,7 @@ limitations under the License. #include "tensorflow/compiler/xla/util.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/cuda_libdevice_path.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" @@ -84,6 +85,8 @@ namespace gpu { namespace { +using tensorflow::strings::StrCat; + // Any address of a variable residing in global memory or returned by one of the // memory allocation routines from the driver or runtime API is always aligned // to at least 256 bytes. @@ -148,6 +151,7 @@ tensorflow::Status OptimizeHloModule( /*is_layout_sensitive=*/false, [](const Shape&, const Shape&) { return false; }); pass.AddPass(); + pass.AddPass(); pass.AddPass(); pass.AddPass(); } @@ -223,7 +227,7 @@ tensorflow::Status PrepareHloModuleForIrEmitting( } // Invokes the ptxas tool on the given PTX string, and dumps its output. -void DumpPtxasInfo(const string& ptx) { +void DumpPtxasInfo(const string& ptx, int cc_major, int cc_minor) { const string ptxas_path = tensorflow::io::JoinPath(tensorflow::CudaRoot(), "bin/ptxas"); // Do not log PTX stats if ptxas is not found at the given path. @@ -245,17 +249,22 @@ void DumpPtxasInfo(const string& ptx) { // Invoke ptxas and collect its output. tensorflow::SubProcess ptxas_info_dumper; - ptxas_info_dumper.SetProgram(ptxas_path, {ptxas_path, ptx_path, "-o", - "/dev/null", "-v", "-arch=sm_35"}); + ptxas_info_dumper.SetProgram(ptxas_path, + {ptxas_path, ptx_path, "-o", "/dev/null", "-v", + StrCat("-arch=sm_", cc_major, cc_minor)}); ptxas_info_dumper.SetChannelAction(tensorflow::CHAN_STDERR, tensorflow::ACTION_PIPE); - CHECK(ptxas_info_dumper.Start()); + if (!ptxas_info_dumper.Start()) { + LOG(ERROR) << "Failed to launch ptxas."; + return; + } string stderr_output; int exit_status = ptxas_info_dumper.Communicate( /*stdin_input=*/nullptr, /*stdout_output=*/nullptr, &stderr_output); XLA_LOG_LINES(tensorflow::INFO, stderr_output); if (exit_status != 0) { - LOG(FATAL) << "Invalid PTX. See the error message above for reasons."; + LOG(ERROR) << "ptxas exited with non-zero error code " << exit_status + << "."; } } @@ -310,12 +319,12 @@ StatusOr> GpuCompiler::Compile( // print one ourselves. XLA_VLOG_LINES(2, buffer_assignment->ToString()); - const string dump_debug_json_to = - module->config().debug_options().xla_dump_debug_json_to(); - if (!dump_debug_json_to.empty()) { + const string xla_dump_hlo_proto_to = + module->config().debug_options().xla_dump_hlo_proto_to(); + if (!xla_dump_hlo_proto_to.empty()) { HloProto proto = MakeHloProto(*module, *buffer_assignment); - TF_RETURN_IF_ERROR(protobuf_util::DumpJsonToDirectory( - proto, dump_debug_json_to, module->name())); + TF_RETURN_IF_ERROR(protobuf_util::DumpProtoToDirectory( + proto, xla_dump_hlo_proto_to, module->name())); } IrEmitterContext ir_emitter_context(module.get(), buffer_assignment.get(), @@ -387,7 +396,7 @@ StatusOr> GpuCompiler::Compile( VLOG(2) << "PTX:"; XLA_VLOG_LINES(2, *ptx); if (VLOG_IS_ON(2)) { - DumpPtxasInfo(*ptx); + DumpPtxasInfo(*ptx, cc_major, cc_minor); } auto thunk_schedule = MakeUnique( diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index 152d226ab05ebb7342483ac127bb6ee16913face..163a161353fdb90cee2968269d572b8414855551 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -102,7 +102,7 @@ void HloToIrBindings::EmitBasePointersForHlos( slice_result.ConsumeValueOrDie(); if (slice.allocation()->is_thread_local()) { llvm::Type* pointee_type = - llvm_ir::ShapeToIrType(non_io_hlo->shape(), ir_builder_); + llvm_ir::ShapeToIrType(non_io_hlo->shape(), module_); BindHloToIrValue(*non_io_hlo, ir_builder_->CreateAlloca(pointee_type), index); } else { @@ -124,18 +124,18 @@ llvm::Value* HloToIrBindings::EmitGetTupleElement(const HloInstruction* gte, if (gte->operand(0)->opcode() != HloOpcode::kGetTupleElement) { return llvm_ir::EmitGetTupleElement( gte->shape(), gte->tuple_index(), /*alignment=*/1, - GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_); + GetTypedIrValue(*gte->operand(0), {}, base_ptr), ir_builder_, module_); } return llvm_ir::EmitGetTupleElement( gte->shape(), gte->tuple_index(), /*alignment=*/1, - EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_); + EmitGetTupleElement(gte->operand(0), base_ptr), ir_builder_, module_); } llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, const ShapeIndex& shape_index, llvm::Value* ir_value) { llvm::Type* pointee_type = llvm_ir::ShapeToIrType( - ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_builder_); + ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_); llvm::Type* dest_type = pointee_type->getPointerTo(); llvm::Value* typed_ir_value; diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h index d43e09e8a8c5cc2efcd8e1fbf9a7c0697e24d73c..a3120f15bcbfb0f2f0bfbd806e7a4ff05316d5dd 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h @@ -36,10 +36,12 @@ class HloToIrBindings { public: HloToIrBindings(const HloModule& module, const BufferAssignment* buffer_assignment, - llvm::IRBuilder<>* ir_builder, bool is_nested) + llvm::IRBuilder<>* ir_builder, llvm::Module* llvm_module, + bool is_nested) : buffer_assignment_(buffer_assignment), is_nested_(is_nested), ir_builder_(ir_builder), + module_(llvm_module), alias_analysis_(module, *buffer_assignment_, &ir_builder_->getContext()) {} @@ -93,6 +95,7 @@ class HloToIrBindings { const bool is_nested_; llvm::IRBuilder<>* ir_builder_; + llvm::Module* module_; // Stores the underlying llvm::IrArray for each HloInstruction. // For an instruction that generates multiple outputs, the root will be a diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc index 3862c2190b1e2df824fa90eafc62bfdfe94e4789..57a3f713e35b506ad9d5caab1ced2c7b74f8efcf 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.cc @@ -53,9 +53,10 @@ namespace gpu { IrEmitter::IrEmitter(const HloModuleConfig& hlo_module_config, IrEmitterContext* ir_emitter_context, bool is_nested) : ir_emitter_context_(ir_emitter_context), - ir_builder_(ir_emitter_context->llvm_module()->getContext()), + module_(ir_emitter_context->llvm_module()), + ir_builder_(module_->getContext()), bindings_(ir_emitter_context->hlo_module(), - &ir_emitter_context->buffer_assignment(), &ir_builder_, + &ir_emitter_context->buffer_assignment(), &ir_builder_, module_, is_nested), hlo_module_config_(hlo_module_config) { ir_builder_.setFastMathFlags(llvm_ir::GetFastMathFlags( @@ -71,18 +72,17 @@ Status IrEmitter::DefaultAction(HloInstruction* hlo) { }; } return EmitTargetElementLoop( - *hlo, GpuElementalIrEmitter(hlo_module_config_, - ir_emitter_context_->llvm_module(), - &ir_builder_, GetNestedComputer()) + *hlo, GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_, + GetNestedComputer()) .MakeElementGenerator(hlo, operand_to_generator)); } -Status IrEmitter::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status IrEmitter::HandleConstant(HloInstruction* constant) { + const Literal& literal = constant->literal(); llvm::Constant* initializer = - llvm_ir::ConvertLiteralToIrConstant(literal, &ir_builder_); + llvm_ir::ConvertLiteralToIrConstant(literal, module_); llvm::GlobalVariable* global_for_const = new llvm::GlobalVariable( - *ir_emitter_context_->llvm_module(), initializer->getType(), + *module_, initializer->getType(), /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, initializer, /*Name=*/""); VLOG(2) << "HandleConstant: " << constant->ToString() << std::endl @@ -106,8 +106,8 @@ Status IrEmitter::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } -Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element) { + auto operand = get_tuple_element->operand(0); CHECK(bindings_.BoundToIrValue(*operand)); bindings_.BindHloToIrValue( *get_tuple_element, @@ -115,32 +115,29 @@ Status IrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, get_tuple_element->shape(), get_tuple_element->tuple_index(), // TODO(b/26344050): tighten the alignment here // based on the real element type. - /*alignment=*/1, GetBasePointer(*operand), &ir_builder_)); + /*alignment=*/1, GetBasePointer(*operand), &ir_builder_, module_)); return Status::OK(); } -Status IrEmitter::HandleSort(HloInstruction* sort, - HloInstruction* operand_instruction) { +Status IrEmitter::HandleSort(HloInstruction*) { // TODO(b/26783907): Implement sort on GPU. return Unimplemented("sort"); } -Status IrEmitter::HandleSend(HloInstruction* send) { +Status IrEmitter::HandleSend(HloInstruction*) { return Unimplemented("Send is not implemented on GPU"); } -Status IrEmitter::HandleRecv(HloInstruction* recv) { +Status IrEmitter::HandleRecv(HloInstruction*) { return Unimplemented("Recv is not implemented on GPU"); } -Status IrEmitter::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitter::HandleTuple(HloInstruction* tuple) { std::vector base_ptrs; - for (const HloInstruction* operand : operands) { + for (const HloInstruction* operand : tuple->operands()) { base_ptrs.push_back(GetBasePointer(*operand)); } - llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_); + llvm_ir::EmitTuple(GetIrArray(*tuple), base_ptrs, &ir_builder_, module_); return Status::OK(); } @@ -321,15 +318,16 @@ Status IrEmitter::EmitAtomicOperationForNestedComputation( return Status::OK(); } -Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { +Status IrEmitter::HandleSelect(HloInstruction* select) { + auto pred = select->operand(0); + auto on_true = select->operand(1); + auto on_false = select->operand(2); TF_RET_CHECK(pred->shape().element_type() == PRED); if (ShapeUtil::IsTuple(select->shape())) { llvm_ir::EmitTupleSelect(GetIrArray(*select), GetIrArray(*pred), GetBasePointer(*on_true), - GetBasePointer(*on_false), &ir_builder_); + GetBasePointer(*on_false), &ir_builder_, module_); return Status::OK(); } @@ -339,9 +337,9 @@ Status IrEmitter::HandleSelect(HloInstruction* select, HloInstruction* pred, return IrEmitter::DefaultAction(select); } -Status IrEmitter::HandleDot(HloInstruction* dot, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) { +Status IrEmitter::HandleDot(HloInstruction* dot) { + auto lhs_instruction = dot->operand(0); + auto rhs_instruction = dot->operand(1); const llvm_ir::IrArray& target_array = GetIrArray(*dot); const llvm_ir::IrArray& lhs_array = GetIrArray(*lhs_instruction); const llvm_ir::IrArray& rhs_array = GetIrArray(*rhs_instruction); @@ -355,7 +353,26 @@ Status IrEmitter::HandleDot(HloInstruction* dot, lhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_); llvm::Value* rhs_value = rhs_array.EmitReadArrayElement(/*index=*/{}, &ir_builder_); - llvm::Value* result = ir_builder_.CreateFMul(lhs_value, rhs_value); + llvm::Value* result; + if (ShapeUtil::ElementIsComplex(lhs_shape)) { + auto real = [&](llvm::Value* x) { + return ir_builder_.CreateExtractValue(x, {0}); + }; + auto imag = [&](llvm::Value* x) { + return ir_builder_.CreateExtractValue(x, {1}); + }; + llvm::Value* real_result = ir_builder_.CreateFSub( + ir_builder_.CreateFMul(real(lhs_value), real(rhs_value)), + ir_builder_.CreateFMul(imag(lhs_value), imag(rhs_value))); + llvm::Value* imag_result = ir_builder_.CreateFAdd( + ir_builder_.CreateFMul(real(lhs_value), imag(rhs_value)), + ir_builder_.CreateFMul(imag(lhs_value), real(rhs_value))); + result = llvm::ConstantAggregateZero::get(lhs_array.GetElementLlvmType()); + result = ir_builder_.CreateInsertValue(result, real_result, {0}); + result = ir_builder_.CreateInsertValue(result, imag_result, {1}); + } else { + result = ir_builder_.CreateFMul(lhs_value, rhs_value); + } target_array.EmitWriteArrayElement(/*index=*/{}, result, &ir_builder_); return Status::OK(); } @@ -411,8 +428,8 @@ Status IrEmitter::HandleDot(HloInstruction* dot, // Initialize the accumulator in the preheader to zero. new llvm::StoreInst( - llvm::ConstantFP::get(accum_type, 0.0), // The value stored. - accum_address, // The address. + llvm::Constant::getNullValue(lhs_array.GetElementLlvmType()), // init 0 + accum_address, // The address. reduction_loop->GetPreheaderBasicBlock() ->getTerminator()); // The instruction this store is inserted before. @@ -427,9 +444,27 @@ Status IrEmitter::HandleDot(HloInstruction* dot, lhs_array.EmitReadArrayElement(lhs_index, &ir_builder_); llvm::Value* rhs_element = rhs_array.EmitReadArrayElement(rhs_index, &ir_builder_); - llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element); llvm::Value* accum = ir_builder_.CreateLoad(accum_address); - llvm::Value* updated_accum = ir_builder_.CreateFAdd(accum, product); + llvm::Value* updated_accum; + if (ShapeUtil::ElementIsComplex(lhs_shape)) { +#define REAL(x) ir_builder_.CreateExtractValue(x, {0}) +#define IMAG(x) ir_builder_.CreateExtractValue(x, {1}) + llvm::Value* product_real = ir_builder_.CreateFSub( + ir_builder_.CreateFMul(REAL(lhs_element), REAL(rhs_element)), + ir_builder_.CreateFMul(IMAG(lhs_element), IMAG(rhs_element))); + llvm::Value* product_imag = ir_builder_.CreateFAdd( + ir_builder_.CreateFMul(REAL(lhs_element), IMAG(rhs_element)), + ir_builder_.CreateFMul(IMAG(lhs_element), REAL(rhs_element))); + updated_accum = ir_builder_.CreateInsertValue( + accum, ir_builder_.CreateFAdd(REAL(accum), product_real), {0}); + updated_accum = ir_builder_.CreateInsertValue( + updated_accum, ir_builder_.CreateFAdd(IMAG(accum), product_imag), {1}); +#undef IMAG +#undef REAL + } else { + llvm::Value* product = ir_builder_.CreateFMul(lhs_element, rhs_element); + updated_accum = ir_builder_.CreateFAdd(accum, product); + } ir_builder_.CreateStore(updated_accum, accum_address); // After the reduction loop exits, store the accumulator into the target @@ -461,10 +496,7 @@ Status IrEmitter::HandleDot(HloInstruction* dot, return Status::OK(); } -Status IrEmitter::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction, - const Window& window) { +Status IrEmitter::HandleConvolution(HloInstruction* convolution) { if (ShapeUtil::HasZeroElements(convolution->shape())) { // Emit no code for an empty output. return Status::OK(); @@ -484,17 +516,18 @@ Status IrEmitter::HandleParameter(HloInstruction* parameter) { return Status::OK(); } -Status IrEmitter::HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) { +Status IrEmitter::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->operand(0); + auto init_value = reduce->operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); return EmitTargetElementLoop( *reduce, [=](const llvm_ir::IrArray::Index& index) -> StatusOr { // Initialize an accumulator with init_value. llvm::AllocaInst* accumulator_addr = ir_builder_.CreateAlloca(llvm_ir::PrimitiveTypeToIrType( - reduce->shape().element_type(), &ir_builder_)); + reduce->shape().element_type(), module_)); ir_builder_.CreateStore( ir_builder_.CreateLoad(GetBasePointer(*init_value)), accumulator_addr); @@ -547,8 +580,7 @@ Status IrEmitter::HandleFusion(HloInstruction* fusion) { for (HloInstruction* operand : fusion->operands()) { parameter_arrays.push_back(GetIrArray(*operand)); } - GpuElementalIrEmitter elemental_emitter(hlo_module_config_, - ir_emitter_context_->llvm_module(), + GpuElementalIrEmitter elemental_emitter(hlo_module_config_, module_, &ir_builder_, GetNestedComputer()); FusedIrEmitter fused_emitter(parameter_arrays, &elemental_emitter); TF_RETURN_IF_ERROR(fusion->fused_expression_root()->Accept(&fused_emitter)); @@ -565,23 +597,19 @@ Status IrEmitter::HandleCall(HloInstruction* call) { GetBasePointer(*call)); } -Status IrEmitter::HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) { +Status IrEmitter::HandleCustomCall(HloInstruction*) { return Unimplemented("custom-call"); } -Status IrEmitter::HandleInfeed(HloInstruction* infeed) { +Status IrEmitter::HandleInfeed(HloInstruction*) { return Unimplemented("Infeed is not supported on GPU (b/30467474)."); } -Status IrEmitter::HandleOutfeed(HloInstruction* outfeed) { +Status IrEmitter::HandleOutfeed(HloInstruction*) { return Unimplemented("Outfeed is not supported on GPU (b/34359662)."); } -Status IrEmitter::HandleRng(HloInstruction* random, - RandomDistribution /*distribution*/) { +Status IrEmitter::HandleRng(HloInstruction* random) { ElementalIrEmitter::HloToElementGeneratorMap operand_to_generator; for (const HloInstruction* operand : random->operands()) { operand_to_generator[operand] = [=](const llvm_ir::IrArray::Index& index) { @@ -591,9 +619,8 @@ Status IrEmitter::HandleRng(HloInstruction* random, // Emits a single-threaded loop because the loop body generated by the element // generator for Rng can't be parallelized (b/32333178). return llvm_ir::LoopEmitter( - GpuElementalIrEmitter(hlo_module_config_, - ir_emitter_context_->llvm_module(), - &ir_builder_, GetNestedComputer()) + GpuElementalIrEmitter(hlo_module_config_, module_, &ir_builder_, + GetNestedComputer()) .MakeElementGenerator(random, operand_to_generator), GetIrArray(*random), &ir_builder_) .EmitLoop(IrName(random)); @@ -634,7 +661,7 @@ StatusOr IrEmitter::ComputeNestedElement( tensorflow::gtl::ArraySlice parameter_elements) { llvm::Value* return_buffer = llvm_ir::EmitAllocaAtFunctionEntry( llvm_ir::PrimitiveTypeToIrType( - computation.root_instruction()->shape().element_type(), &ir_builder_), + computation.root_instruction()->shape().element_type(), module_), "return_buffer", &ir_builder_); std::vector parameter_buffers; for (llvm::Value* parameter_element : parameter_elements) { diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter.h b/tensorflow/compiler/xla/service/gpu/ir_emitter.h index 5e3f3bfdf18bdd5b4f8d0e565d1bb2613cebc3a1..263992d92544166c0d08a6c60b43e78f10f06aed 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter.h @@ -74,39 +74,25 @@ class IrEmitter : public DfsHloVisitorWithDefault { // The following methods implement the DfsHloVisitorWithDefault interface. Status DefaultAction(HloInstruction* hlo) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; Status HandleBitcast(HloInstruction* bitcast) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleDot(HloInstruction* dot) override; + Status HandleConvolution(HloInstruction* convolution) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; Status HandleInfeed(HloInstruction* infeed) override; Status HandleOutfeed(HloInstruction* outfeed) override; - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override; + Status HandleSort(HloInstruction* sort) override; Status HandleSend(HloInstruction* send) override; Status HandleRecv(HloInstruction* recv) override; Status HandleParameter(HloInstruction* parameter) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleReduce(HloInstruction* reduce) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleSelect(HloInstruction* select) override; Status HandleFusion(HloInstruction* fusion) override; Status HandleCall(HloInstruction* call) override; - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override; - Status HandleRng(HloInstruction* random, - RandomDistribution /*distribution*/) override; + Status HandleCustomCall(HloInstruction* custom_call) override; + Status HandleRng(HloInstruction* random) override; Status FinishVisit(HloInstruction* root) override { return Status::OK(); } @@ -162,6 +148,7 @@ class IrEmitter : public DfsHloVisitorWithDefault { } IrEmitterContext* ir_emitter_context_; + llvm::Module* module_; // The following fields track the IR emission state. According to LLVM memory // management rules, their memory is owned by the module. @@ -232,28 +219,17 @@ class IrEmitterUnnested : public IrEmitter { // IrEmitterUnnested handles the following instructions differently from // IrEmitter. Status HandleCopy(HloInstruction* copy) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) override; + Status HandleConvolution(HloInstruction* convolution) override; + Status HandleDot(HloInstruction* dot) override; Status HandleFusion(HloInstruction* fusion) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleReduce(HloInstruction* reduce) override; Status HandleSelectAndScatter(HloInstruction* instruction) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; + Status HandleTuple(HloInstruction* tuple) override; Status HandleWhile(HloInstruction* xla_while) override; Status HandleInfeed(HloInstruction* xla_infeed) override; - Status HandleRng(HloInstruction* random, - RandomDistribution distribution) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleRng(HloInstruction* random) override; + Status HandleSelect(HloInstruction* select) override; Status EmitTargetElementLoop( const HloInstruction& hlo, @@ -339,8 +315,12 @@ class IrEmitterUnnested : public IrEmitter { // to make sure `inst` outlives the lifetime of the returned Thunk object. std::unique_ptr BuildGemmThunk(const HloInstruction* inst); - // Returns a CopyThunk that calls host-to-device cuMemcpy to implement `inst`. - std::unique_ptr BuildCopyThunk(const HloInstruction* inst); + // Returns a thunk that calls host-to-device cuMemcpy to implement `inst`. + std::unique_ptr BuildHostToDeviceCopyThunk(const HloInstruction* inst); + + // Returns a thunk that calls device-to-device cuMemcpy to implement `inst`. + std::unique_ptr BuildDeviceToDeviceCopyThunk( + const HloInstruction* inst); // Returns an InfeedThunk that performs device-to-device memcpy to implement // `inst`. diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc index 57f010530cc93cf5f2ef60470ce416fe9333a94e..5da1a130d5654b86803396b07a6501c59a182c67 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_nested.cc @@ -52,9 +52,9 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation( io_hlos->push_back(param); const Shape& param_shape = param->shape(); argument_types.push_back( - llvm_ir::ShapeToIrType(param_shape, &ir_builder_)->getPointerTo()); - int64 param_size = llvm_ir::ByteSizeOf( - param_shape, ir_emitter_context_->llvm_module()->getDataLayout()); + llvm_ir::ShapeToIrType(param_shape, module_)->getPointerTo()); + int64 param_size = + llvm_ir::ByteSizeOf(param_shape, module_->getDataLayout()); argument_dereferenceable_bytes.push_back(param_size); } { @@ -62,7 +62,7 @@ llvm::Function* IrEmitterNested::EmitBasePointersForNestedComputation( io_hlos->push_back(root); const Shape& root_shape = root->shape(); argument_types.push_back( - llvm_ir::ShapeToIrType(root_shape, &ir_builder_)->getPointerTo()); + llvm_ir::ShapeToIrType(root_shape, module_)->getPointerTo()); int64 root_size = llvm_ir::ByteSizeOf( root_shape, ir_emitter_context_->llvm_module()->getDataLayout()); argument_dereferenceable_bytes.push_back(root_size); diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 120d50ed2582c43816a5e2ac757710cff13f43b7..7b4662fc80c5518135c827489a3724e477b2bad1 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -146,7 +146,7 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { } namespace { -bool ImplementedAsMemcpy(const HloInstruction& hlo) { +bool ImplementedAsHostToDeviceMemcpy(const HloInstruction& hlo) { // `hlo` needs to satisfy three conditions to be implemented as a // host-to-device cuMemcpy. // @@ -157,6 +157,20 @@ bool ImplementedAsMemcpy(const HloInstruction& hlo) { hlo.operand(0)->opcode() == HloOpcode::kConstant && ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()); } + +bool ImplementedAsDeviceToDeviceMemcpy( + const BufferAssignment& buffer_assignment, const HloInstruction& hlo) { + // `hlo` needs to satisfy three conditions to be implemented as a + // device-to-device cuMemcpy. + // + // 1. `hlo` is a kCopy instruction. + // 2. `hlo` and its operand have the same shape (thus the same layout too). + // 3. The operand to `hlo` has a buffer assignment (constants do not, for + // instance) which means the source buffer also resides on the device. + return hlo.opcode() == HloOpcode::kCopy && + ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()) && + buffer_assignment.HasTopLevelAllocation(hlo.operand(0)); +} } // namespace llvm::Function* IrEmitterUnnested::BuildKernelPrototype( @@ -231,28 +245,22 @@ Status IrEmitterUnnested::DefaultAction(HloInstruction* hlo) { return IrEmitter::DefaultAction(hlo); } -Status IrEmitterUnnested::HandleDot(HloInstruction* dot, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) { +Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { if (ImplementedAsGemm(*dot)) { thunk_sequence_->emplace_back(BuildGemmThunk(dot)); return Status::OK(); } thunk_sequence_->emplace_back(BuildKernelThunk(dot)); - return IrEmitter::HandleDot(dot, lhs_instruction, rhs_instruction); + return IrEmitter::HandleDot(dot); } -Status IrEmitterUnnested::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction, - const Window& window) { +Status IrEmitterUnnested::HandleConvolution(HloInstruction* convolution) { if (ImplementedAsDnnConvolution(*convolution)) { thunk_sequence_->emplace_back(BuildConvolutionThunk(convolution)); return Status::OK(); } thunk_sequence_->emplace_back(BuildKernelThunk(convolution)); - return IrEmitter::HandleConvolution(convolution, lhs_instruction, - rhs_instruction, window); + return IrEmitter::HandleConvolution(convolution); } Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) { @@ -664,8 +672,13 @@ int64 EmitTranspose021Tiled(llvm_ir::IrArray input, llvm_ir::IrArray output, } // namespace Status IrEmitterUnnested::HandleCopy(HloInstruction* copy) { - if (ImplementedAsMemcpy(*copy)) { - thunk_sequence_->emplace_back(BuildCopyThunk(copy)); + if (ImplementedAsHostToDeviceMemcpy(*copy)) { + thunk_sequence_->emplace_back(BuildHostToDeviceCopyThunk(copy)); + return Status::OK(); + } + if (ImplementedAsDeviceToDeviceMemcpy( + ir_emitter_context_->buffer_assignment(), *copy)) { + thunk_sequence_->emplace_back(BuildDeviceToDeviceCopyThunk(copy)); return Status::OK(); } bool is_transpose_021; @@ -738,8 +751,8 @@ Status IrEmitterUnnested::EmitColumnReduction( auto loop_body_emitter = [=](const llvm_ir::IrArray::Index& tile_index) -> Status { // Emit the loop body that reduces one tile. - llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType( - input_shape.element_type(), &ir_builder_); + llvm::Type* element_ir_type = + llvm_ir::PrimitiveTypeToIrType(input_shape.element_type(), module_); llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca( element_ir_type, /*ArraySize=*/nullptr, "partial_reduction_result"); { @@ -954,7 +967,7 @@ Status IrEmitterUnnested::EmitRowReduction( [=](const llvm_ir::IrArray::Index& tile_index) -> Status { // Emit the loop body that reduces one tile. llvm::Type* element_ir_type = llvm_ir::PrimitiveTypeToIrType( - input_shape.element_type(), &ir_builder_); + input_shape.element_type(), ir_emitter_context_->llvm_module()); llvm::Value* partial_reduction_result_address = ir_builder_.CreateAlloca( element_ir_type, /*ArraySize=*/nullptr, "partial_reduction_result"); { @@ -1215,10 +1228,11 @@ Status IrEmitterUnnested::EmitReductionToVector( } } -Status IrEmitterUnnested::HandleReduce( - HloInstruction* reduce, HloInstruction* input, HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions_to_reduce, - HloComputation* reducer) { +Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) { + auto input = reduce->operand(0); + auto init_value = reduce->operand(1); + tensorflow::gtl::ArraySlice dimensions_to_reduce(reduce->dimensions()); + HloComputation* reducer = reduce->to_apply(); // HandleReduce specializes reduction from a multi-dimensional array to a 1D // array. The specialized version requires an initializer thunk that // initializes the output array to the initial value of the reduce. @@ -1246,13 +1260,11 @@ Status IrEmitterUnnested::HandleReduce( } thunk_sequence_->emplace_back(BuildKernelThunk(reduce)); - return IrEmitter::HandleReduce(reduce, input, init_value, - dimensions_to_reduce, reducer); + return IrEmitter::HandleReduce(reduce); } -Status IrEmitterUnnested::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status IrEmitterUnnested::HandleTuple(HloInstruction* tuple) { + tensorflow::gtl::ArraySlice operands(tuple->operands()); bool all_tuple_elements_have_buffer = std::all_of( operands.begin(), operands.end(), [this](HloInstruction* tuple_element) { return ir_emitter_context_->buffer_assignment().HasTopLevelAllocation( @@ -1277,11 +1289,10 @@ Status IrEmitterUnnested::HandleTuple( return Status::OK(); } thunk_sequence_->emplace_back(BuildKernelThunk(tuple)); - return IrEmitter::HandleTuple(tuple, operands); + return IrEmitter::HandleTuple(tuple); } -Status IrEmitterUnnested::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { +Status IrEmitterUnnested::HandleGetTupleElement(HloInstruction*) { // GetTupleElement IR is emitted in the IR context of the user instruction, // and so we do not build a kernel for GetTupleElement instructions. return Status::OK(); @@ -1341,7 +1352,8 @@ Status IrEmitterUnnested::HandleSelectAndScatter( // boolean flag if the value is initialized. The initialized_flag is set // false. llvm::Value* selected_value_address = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(operand_element_type, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(operand_element_type, + ir_emitter_context_->llvm_module()), "selected_value_address", &ir_builder_); llvm::Value* selected_index_address = llvm_ir::EmitAllocaAtFunctionEntryWithCount( @@ -1421,7 +1433,8 @@ Status IrEmitterUnnested::HandleSelectAndScatter( llvm::Value* operand_address = operand_array.EmitArrayElementAddress(operand_index, &ir_builder_); llvm::Value* select_return_buffer = llvm_ir::EmitAllocaAtFunctionEntry( - llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), + llvm_ir::PrimitiveTypeToIrType(PRED, + ir_emitter_context_->llvm_module()), "select_return_buffer", &ir_builder_); TF_RETURN_IF_ERROR(EmitCallToNestedComputation( *select_and_scatter->select(), @@ -1431,8 +1444,10 @@ Status IrEmitterUnnested::HandleSelectAndScatter( // If the 'select' function returns false, update the selected value and the // index to the currently visiting operand. llvm::Value* cond = ir_builder_.CreateICmpNE( - result, llvm::ConstantInt::get( - llvm_ir::PrimitiveTypeToIrType(PRED, &ir_builder_), 0), + result, + llvm::ConstantInt::get(llvm_ir::PrimitiveTypeToIrType( + PRED, ir_emitter_context_->llvm_module()), + 0), "boolean_predicate"); llvm_ir::LlvmIfData if_select_lhs = llvm_ir::EmitIfThenElse(cond, "if-select-lhs", &ir_builder_); @@ -1502,18 +1517,14 @@ Status IrEmitterUnnested::HandleWhile(HloInstruction* xla_while) { return Status::OK(); } -Status IrEmitterUnnested::HandleRng(HloInstruction* random, - RandomDistribution distribution) { +Status IrEmitterUnnested::HandleRng(HloInstruction* random) { thunk_sequence_->push_back(BuildKernelThunk(random)); - return IrEmitter::HandleRng(random, distribution); + return IrEmitter::HandleRng(random); } -Status IrEmitterUnnested::HandleSelect(HloInstruction* select, - HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { +Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { thunk_sequence_->push_back(BuildKernelThunk(select)); - return IrEmitter::HandleSelect(select, pred, on_true, on_false); + return IrEmitter::HandleSelect(select); } Status IrEmitterUnnested::HandleInfeed(HloInstruction* infeed) { @@ -1579,11 +1590,11 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( llvm_ir::AsString(kernel->getName()), inst); } -std::unique_ptr IrEmitterUnnested::BuildCopyThunk( +std::unique_ptr IrEmitterUnnested::BuildHostToDeviceCopyThunk( const HloInstruction* inst) { const HloInstruction* operand = inst->operand(0); CHECK_EQ(HloOpcode::kConstant, operand->opcode()); - return MakeUnique( + return MakeUnique( /*source_address=*/operand->literal().InternalData(), /*destination_buffer=*/GetAllocationSlice(*inst), /*mem_size=*/ @@ -1592,6 +1603,18 @@ std::unique_ptr IrEmitterUnnested::BuildCopyThunk( inst); } +std::unique_ptr IrEmitterUnnested::BuildDeviceToDeviceCopyThunk( + const HloInstruction* inst) { + const HloInstruction* operand = inst->operand(0); + return MakeUnique( + /*source_address=*/GetAllocationSlice(*operand), + /*destination_buffer=*/GetAllocationSlice(*inst), + /*mem_size=*/ + llvm_ir::ByteSizeOf(operand->shape(), + ir_emitter_context_->llvm_module()->getDataLayout()), + inst); +} + std::unique_ptr IrEmitterUnnested::BuildInfeedThunk( const HloInstruction* inst) { CHECK_EQ(HloOpcode::kInfeed, inst->opcode()); @@ -1846,7 +1869,8 @@ Status IrEmitterUnnested::EmitTargetElementLoopInThunk( tuple_operand_ptrs.push_back(output_arrays[i].GetBasePointer()); } ir_builder_.SetInsertPoint(ir_builder_.GetInsertBlock()->getTerminator()); - llvm_ir::EmitTuple(GetIrArray(hlo), tuple_operand_ptrs, &ir_builder_); + llvm_ir::EmitTuple(GetIrArray(hlo), tuple_operand_ptrs, &ir_builder_, + module_); return Status::OK(); } diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc index 9b3104eaacdbb083db2a55c75fae3e94c8ff282f..72c70b38238eedb67622f4816e1de264f3c9ed4b 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.cc +++ b/tensorflow/compiler/xla/service/hlo_computation.cc @@ -56,7 +56,6 @@ std::unique_ptr HloComputation::Builder::Build( HloInstruction* root = root_instruction ? root_instruction : last_added_instruction_; CHECK_NE(nullptr, root); - return WrapUnique(new HloComputation(name_, parameter_count, &instructions_, root, fusion_instruction_)); } @@ -373,13 +372,14 @@ string HloComputation::ToString(int nested_level) const { for (int i = 0; i < nested_level; i++) { s << " "; } - s << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) - << " { \n"; + s << "%" << name() << " " << ShapeUtil::HumanString(ComputeProgramShape()) + << " {\n"; for (const HloInstruction* instruction : MakeInstructionPostOrder()) { for (int i = 0; i < nested_level; i++) { s << " "; } - s << " " << instruction->ToString() << "\n"; + s << " " << (instruction == root_instruction_ ? "ROOT " : "") + << instruction->ToString() << "\n"; if (instruction->opcode() == HloOpcode::kFusion) { s << instruction->fused_instructions_computation()->ToString( nested_level + 1) @@ -734,6 +734,10 @@ std::unique_ptr HloComputation::Clone(const string& suffix) { } new_instr = instr->CloneWithNewOperands(instr->shape(), new_operands); + new_instr->set_metadata(instr->metadata()); + if (instr->has_sharding()) { + new_instr->set_sharding(instr->sharding()); + } InsertOrDie(&clone_map, instr, new_instr.get()); instructions.push_back(std::move(new_instr)); } diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h index 3515a6b5df2ed9a77bdf611adfbf14536aed8348..f4edd175016ee30d31cc0cad6bdbd3eaa014c704 100644 --- a/tensorflow/compiler/xla/service/hlo_computation.h +++ b/tensorflow/compiler/xla/service/hlo_computation.h @@ -312,8 +312,7 @@ class HloComputation { explicit HloComputation( const string& name, int parameter_count, std::vector>* instructions, - HloInstruction* root_instruction, - HloInstruction* fusion_instruction = nullptr); + HloInstruction* root_instruction, HloInstruction* fusion_instruction); // Internal helper for adding instructions. HloInstruction* AddInstructionInternal( @@ -359,11 +358,6 @@ class HloComputation { std::vector param_instructions_; - // Unique name generator for instruction identifiers. Instruction names should - // be unique per computation and this is enforced when instructions are added - // to the computation. - NameUniquer instruction_name_uniquer_; - TF_DISALLOW_COPY_AND_ASSIGN(HloComputation); }; diff --git a/tensorflow/compiler/xla/service/hlo_constant_folding.cc b/tensorflow/compiler/xla/service/hlo_constant_folding.cc index b30c7b417f3785bd485f17d7f46a8b47ef4d4b58..53450991b6fad5b9651d9d23b55c908e6b68e5dd 100644 --- a/tensorflow/compiler/xla/service/hlo_constant_folding.cc +++ b/tensorflow/compiler/xla/service/hlo_constant_folding.cc @@ -49,8 +49,8 @@ StatusOr HloConstantFolding::Run(HloModule* module) { continue; } // Skip Constant, Parameter, Reduce operation. - // TODO(b/35975797): Enable Reduce operation once arbitary computation are - // supported by the evaluator. + // TODO(b/35975797): Enable Reduce operation once arbitrary computation + // are supported by the evaluator. // TODO(b/64407269): Enable Tuple once the timeout issue is resolved. if (instruction->opcode() == HloOpcode::kParameter || instruction->opcode() == HloOpcode::kConstant || @@ -63,8 +63,8 @@ StatusOr HloConstantFolding::Run(HloModule* module) { continue; } - // Broadcasts dramatically increase the size of constants with is often - // detrimental to performance and memory capacity so do not fold + // Broadcasts dramatically increase the size of constants, which is often + // detrimental to performance and memory capacity, so do not fold // broadcasts. if (instruction->opcode() == HloOpcode::kBroadcast) { continue; diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc index 84d55d4b5f83bd54940d3011037598deb6ec934b..ab018c4cf2da770eabe74d7b5a670a19937b1b9a 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc @@ -126,16 +126,11 @@ Status HloCostAnalysis::HandleElementwiseBinary(HloInstruction* hlo) { return HandleElementwiseOp(hlo); } -Status HloCostAnalysis::HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, - HloInstruction* rhs) { +Status HloCostAnalysis::HandleCompare(HloInstruction* compare) { return HandleElementwiseOp(compare); } -Status HloCostAnalysis::HandleClamp(HloInstruction* clamp, - HloInstruction* min_instruction, - HloInstruction* arg_instruction, - HloInstruction* max_instruction) { +Status HloCostAnalysis::HandleClamp(HloInstruction* clamp) { return HandleElementwiseOp(clamp); } @@ -143,57 +138,38 @@ Status HloCostAnalysis::HandleReducePrecision(HloInstruction* hlo) { return HandleElementwiseOp(hlo); } -Status HloCostAnalysis::HandleParameter(HloInstruction* parameter) { +Status HloCostAnalysis::HandleParameter(HloInstruction*) { current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status HloCostAnalysis::HandleConstant(HloInstruction*) { current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status HloCostAnalysis::HandleGetTupleElement(HloInstruction*) { // GetTupleElement forwards a pointer and does not touch each element in the // output. current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleSelect(HloInstruction* select, - HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) { - return Status::OK(); -} +Status HloCostAnalysis::HandleSelect(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleReverse(HloInstruction* reverse, - HloInstruction* operand_instruction) { - return Status::OK(); -} +Status HloCostAnalysis::HandleReverse(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleSlice(HloInstruction* slice, - HloInstruction* operand_instruction) { - return Status::OK(); -} +Status HloCostAnalysis::HandleSlice(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) { +Status HloCostAnalysis::HandleDynamicSlice(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleDynamicUpdateSlice( - HloInstruction* dynamic_update, HloInstruction* operand, - HloInstruction* update, HloInstruction* start_indices) { +Status HloCostAnalysis::HandleDynamicUpdateSlice(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status HloCostAnalysis::HandleTuple(HloInstruction* tuple) { // The tuple instruction only gathers pointers from inputs (it doesn't iterate // through them). The memory touched is then only the size of the output // index table of the tuple. @@ -202,9 +178,7 @@ Status HloCostAnalysis::HandleTuple( return Status::OK(); } -Status HloCostAnalysis::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { +Status HloCostAnalysis::HandleConcatenate(HloInstruction*) { return Status::OK(); } @@ -212,15 +186,11 @@ Status HloCostAnalysis::HandleConvert(HloInstruction* convert) { return HandleElementwiseOp(convert); } -Status HloCostAnalysis::HandleCopy(HloInstruction* copy) { - return Status::OK(); -} +Status HloCostAnalysis::HandleCopy(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleDot(HloInstruction* dot, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction) { - const Shape& lhs_shape = lhs_instruction->shape(); - const Shape& rhs_shape = rhs_instruction->shape(); +Status HloCostAnalysis::HandleDot(HloInstruction* dot) { + const Shape& lhs_shape = dot->operand(0)->shape(); + const Shape& rhs_shape = dot->operand(1)->shape(); // Count of elements along the reduction dimension (last dimension for the // rhs). int64 reduction_width = lhs_shape.dimensions(ShapeUtil::Rank(lhs_shape) - 1); @@ -240,21 +210,14 @@ Status HloCostAnalysis::HandleDot(HloInstruction* dot, return Status::OK(); } -Status HloCostAnalysis::HandleInfeed(HloInstruction* infeed) { - return Status::OK(); -} +Status HloCostAnalysis::HandleInfeed(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleOutfeed(HloInstruction* outfeed) { - return Status::OK(); -} +Status HloCostAnalysis::HandleOutfeed(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleMap( - HloInstruction* map, tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice /*static_operands*/) { +Status HloCostAnalysis::HandleMap(HloInstruction* map) { // Compute properties of the mapped function. TF_ASSIGN_OR_RETURN(const Properties sub_properties, - ProcessSubcomputation(function)); + ProcessSubcomputation(map->to_apply())); // Compute the cost of all elements for this Map operation. const int64 element_count = ShapeUtil::ElementsIn(map->shape()); @@ -266,9 +229,9 @@ Status HloCostAnalysis::HandleMap( return Status::OK(); } -Status HloCostAnalysis::HandleReduce( - HloInstruction* reduce, HloInstruction* arg, HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, HloComputation* function) { +Status HloCostAnalysis::HandleReduce(HloInstruction* reduce) { + auto arg = reduce->operand(0); + HloComputation* function = reduce->to_apply(); // Compute the cost of the user function. TF_ASSIGN_OR_RETURN(const Properties sub_properties, ProcessSubcomputation(function)); @@ -284,10 +247,9 @@ Status HloCostAnalysis::HandleReduce( return Status::OK(); } -Status HloCostAnalysis::HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, - const Window& window, - HloComputation* function) { +Status HloCostAnalysis::HandleReduceWindow(HloInstruction* reduce_window) { + const Window& window = reduce_window->window(); + auto function = reduce_window->to_apply(); // Compute the properties of the reduction function. TF_ASSIGN_OR_RETURN(const Properties sub_properties, ProcessSubcomputation(function)); @@ -342,55 +304,45 @@ Status HloCostAnalysis::HandleSelectAndScatter(HloInstruction* instruction) { return Status::OK(); } -Status HloCostAnalysis::HandleBitcast(HloInstruction* bitcast) { +Status HloCostAnalysis::HandleBitcast(HloInstruction*) { // A bitcast does no computation and touches no memory. current_properties_[kBytesAccessedKey] = 0; return Status::OK(); } -Status HloCostAnalysis::HandleBroadcast(HloInstruction* broadcast) { +Status HloCostAnalysis::HandleBroadcast(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandlePad(HloInstruction* pad) { return Status::OK(); } +Status HloCostAnalysis::HandlePad(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleSend(HloInstruction* send) { - return Status::OK(); -} +Status HloCostAnalysis::HandleSend(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleRecv(HloInstruction* recv) { - return Status::OK(); -} +Status HloCostAnalysis::HandleRecv(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleReshape(HloInstruction* reshape) { - return Status::OK(); -} +Status HloCostAnalysis::HandleReshape(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleBatchNormTraining( - HloInstruction* batch_norm_training) { +Status HloCostAnalysis::HandleBatchNormTraining(HloInstruction*) { // TODO(b/62294698): Implement cost analysis for batch-norm-training. return Status::OK(); } -Status HloCostAnalysis::HandleBatchNormInference( - HloInstruction* batch_norm_inference) { +Status HloCostAnalysis::HandleBatchNormInference(HloInstruction*) { // TODO(b/62294698): Implement cost analysis for batch-norm-inference. return Status::OK(); } -Status HloCostAnalysis::HandleBatchNormGrad(HloInstruction* batch_norm_grad) { +Status HloCostAnalysis::HandleBatchNormGrad(HloInstruction*) { // TODO(b/62294698): Implement cost analysis for batch-norm-grad. return Status::OK(); } -Status HloCostAnalysis::HandleTranspose(HloInstruction* transpose) { +Status HloCostAnalysis::HandleTranspose(HloInstruction*) { return Status::OK(); } -Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution, - HloInstruction* lhs_instruction, - HloInstruction* rhs_instruction, - const Window& window) { +Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution) { + auto rhs_instruction = convolution->operand(1); const auto& dnums = convolution->convolution_dimension_numbers(); const int64 output_features = convolution->shape().dimensions(dnums.output_feature_dimension()); @@ -398,7 +350,9 @@ Status HloCostAnalysis::HandleConvolution(HloInstruction* convolution, // For each output element, we do one fma per element in the kernel at some // given output feature index. const int64 fmas_per_output_element = - ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features; + output_features > 0 + ? ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features + : 0; const int64 output_elements = ShapeUtil::ElementsIn(convolution->shape()); current_properties_[kFlopsKey] = output_elements * fmas_per_output_element * kFmaFlops; @@ -415,8 +369,7 @@ Status HloCostAnalysis::HandleCrossReplicaSum(HloInstruction* crs) { return Status::OK(); } -Status HloCostAnalysis::HandleRng(HloInstruction* random, - RandomDistribution distribution) { +Status HloCostAnalysis::HandleRng(HloInstruction* random) { // TODO(b/26346211): Implement better estimates for the RNG cost, since the // cost changes with the implementation and the distribution. For now, assume // the cost of each RNG is same as a transcendental operation. @@ -460,18 +413,14 @@ Status HloCostAnalysis::HandleCall(HloInstruction* call) { return Status::OK(); } -Status HloCostAnalysis::HandleCustomCall( - HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) { +Status HloCostAnalysis::HandleCustomCall(HloInstruction*) { return Unimplemented("Custom-call is not implemented for HLO cost analysis."); } -Status HloCostAnalysis::HandleSort(HloInstruction* sort, - HloInstruction* operand_instruction) { +Status HloCostAnalysis::HandleSort(HloInstruction* sort) { // This assumes a comparison based N*log(N) algorithm. As for all ops, the // actual properties of the op depend on the backend implementation. - int64 elements = ShapeUtil::ElementsIn(operand_instruction->shape()); + int64 elements = ShapeUtil::ElementsIn(sort->operand(0)->shape()); current_properties_[kFlopsKey] = elements * tensorflow::Log2Ceiling(elements); return Status::OK(); } @@ -500,9 +449,7 @@ Status HloCostAnalysis::HandleWhile(HloInstruction* xla_while) { return Status::OK(); } -Status HloCostAnalysis::FinishVisit(HloInstruction* root) { - return Status::OK(); -} +Status HloCostAnalysis::FinishVisit(HloInstruction*) { return Status::OK(); } float HloCostAnalysis::flop_count() const { return GetProperty(kFlopsKey, properties_sum_); diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.h b/tensorflow/compiler/xla/service/hlo_cost_analysis.h index d71c2eccee349835c2f998e1774a4d292181c2e2..93b1b3eb20cf88292d38549016c9a0b662e155ee 100644 --- a/tensorflow/compiler/xla/service/hlo_cost_analysis.h +++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.h @@ -51,70 +51,41 @@ class HloCostAnalysis : public DfsHloVisitor { Status HandleElementwiseUnary(HloInstruction* hlo) override; Status HandleElementwiseBinary(HloInstruction* hlo) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; - Status HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) override; - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override; + Status HandleConstant(HloInstruction* constant) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; + Status HandleSelect(HloInstruction* select) override; + Status HandleCompare(HloInstruction* compare) override; + Status HandleClamp(HloInstruction* clamp) override; Status HandleReducePrecision(HloInstruction* hlo) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; Status HandleSend(HloInstruction* send) override; Status HandleRecv(HloInstruction* recv) override; Status HandleConvert(HloInstruction* convert) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override; - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override; + Status HandleDot(HloInstruction* dot) override; + Status HandleConvolution(HloInstruction* convolution) override; Status HandleCrossReplicaSum(HloInstruction* crs) override; Status HandleInfeed(HloInstruction* infeed) override; Status HandleOutfeed(HloInstruction* outfeed) override; - Status HandleRng(HloInstruction* random, - RandomDistribution distribution) override; - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override; - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override; + Status HandleRng(HloInstruction* random) override; + Status HandleReverse(HloInstruction* reverse) override; + Status HandleSort(HloInstruction* sort) override; Status HandleParameter(HloInstruction* parameter) override; - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function_handle) override; + Status HandleReduce(HloInstruction* reduce) override; Status HandleBatchNormTraining(HloInstruction* batch_norm_training) override; Status HandleBatchNormInference( HloInstruction* batch_norm_inference) override; Status HandleBatchNormGrad(HloInstruction* batch_norm_grad) override; Status HandleFusion(HloInstruction* fusion) override; Status HandleCall(HloInstruction* call) override; - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override; - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override; - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) override; - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override; - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override; + Status HandleCustomCall(HloInstruction* custom_call) override; + Status HandleSlice(HloInstruction* slice) override; + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override; + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleMap(HloInstruction* map) override; + Status HandleReduceWindow(HloInstruction* reduce_window) override; Status HandleSelectAndScatter(HloInstruction* instruction) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleBroadcast(HloInstruction* broadcast) override; diff --git a/tensorflow/compiler/xla/service/hlo_dce.h b/tensorflow/compiler/xla/service/hlo_dce.h index fca3fa0f58b7c5929c6ffa6c2d8ae6f76660b380..4e244494d6f98c48f4376bd762f116b9a9c2084d 100644 --- a/tensorflow/compiler/xla/service/hlo_dce.h +++ b/tensorflow/compiler/xla/service/hlo_dce.h @@ -24,10 +24,15 @@ limitations under the License. namespace xla { -// HLO pass which removes all dead instructions from each computation in the -// module. An instruction is dead if it is not reachable from the root. This -// pass does not remove dead parameter instructions as parameter instructions -// cannot be deleted, nor does the pass remove dead computations. +// HLO pass which removes dead instructions from each computation in the module +// and removes dead computations from the module. +// +// An instruction is dead if it is not reachable from the root. A computation is +// dead if it is not the entry computation of the module and it is not reachable +// from the entry computation. +// +// This pass does not remove dead parameter instructions, as parameter +// instructions cannot be deleted. class HloDCE : public HloPassInterface { public: ~HloDCE() override {} diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index 5fd891835d7bad0218c1d478f866d97bdf9dd7ca..88b77ccdd03eb129f81cfa1da430e882ea569df4 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -50,6 +50,12 @@ namespace xla { namespace { +template +struct is_complex_t : public std::false_type {}; + +template <> +struct is_complex_t : public std::true_type {}; + template StatusOr> Compare(const Shape& shape, HloOpcode opcode, const Literal& lhs_literal, @@ -101,6 +107,37 @@ StatusOr> Compare(const Shape& shape, HloOpcode opcode, return std::move(result); } +template <> +StatusOr> Compare( + const Shape& shape, HloOpcode opcode, const Literal& lhs_literal, + const Literal& rhs_literal) { + std::function compare_op; + switch (opcode) { + case HloOpcode::kEq: + compare_op = [](complex64 lhs_el, complex64 rhs_el) { + return lhs_el == rhs_el; + }; + break; + case HloOpcode::kNe: + compare_op = [](complex64 lhs_el, complex64 rhs_el) { + return lhs_el != rhs_el; + }; + break; + default: + LOG(FATAL) << "unhandled HLO opcode for conversion to Comparison: " + << HloOpcodeString(opcode); + } + + auto result = Literal::CreateFromShape(shape); + TF_RETURN_IF_ERROR(result->Populate( + [&](tensorflow::gtl::ArraySlice multi_index) { + return compare_op(lhs_literal.Get(multi_index), + rhs_literal.Get(multi_index)); + })); + + return std::move(result); +} + template StatusOr> ElementWiseUnaryOpImpl( HloInstruction* instruction, @@ -138,7 +175,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { Status DefaultAction(HloInstruction* hlo_instruction) override { return Unimplemented("unhandled HLO ops for HloEvaluator: %s.", HloOpcodeString(hlo_instruction->opcode()).c_str()); - }; + } // TODO(b/35950897): many of the stl functions used in the handlers are not // overloaded for every XLA primitive types. @@ -146,7 +183,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { template ::value>::type* = nullptr> - Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { + Status HandleAbs(HloInstruction* abs) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[abs], ElementWiseUnaryOp(abs, [](NativeT elem_operand) { return elem_operand; @@ -156,8 +193,9 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { template < typename NativeT, - typename std::enable_if::value>::type* = nullptr> - Status HandleAbs(HloInstruction* abs, HloInstruction* operand) { + typename std::enable_if::value || + is_complex_t::value>::type* = nullptr> + Status HandleAbs(HloInstruction* abs) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[abs], ElementWiseUnaryOp(abs, [](NativeT elem_operand) { return std::abs(elem_operand); @@ -165,11 +203,14 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleAbs(HloInstruction* abs, HloInstruction* operand) override { - return HandleAbs(abs, operand); + Status HandleAbs(HloInstruction* abs) override { + return HandleAbs(abs); } - Status HandleRound(HloInstruction* round) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRound(HloInstruction* round) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[round], ElementWiseUnaryOp(round, [](ReturnT elem_operand) { return std::round(elem_operand); @@ -177,6 +218,17 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRound(HloInstruction* round) { + return InvalidArgument("Unsupported type for Round"); + } + + Status HandleRound(HloInstruction* round) override { + return HandleRound(round); + } + Status HandleBroadcast(HloInstruction* broadcast) override { parent_->evaluated_[broadcast] = Literal::CreateFromShape(broadcast->shape()); @@ -205,15 +257,29 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } return operand_to_broadcast.Get(broadcast_indices); }); - }; + } - Status HandleCeil(HloInstruction* ceil, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleCeil(HloInstruction* ceil) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[ceil], ElementWiseUnaryOp(ceil, [](ReturnT elem_operand) { return std::ceil(elem_operand); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleCeil(HloInstruction* ceil) { + return InvalidArgument("Unsupported type for Ceil"); + } + + Status HandleCeil(HloInstruction* ceil) override { + return HandleCeil(ceil); + } Status HandleConvert(HloInstruction* convert) override { const HloInstruction* operand = convert->operand(0); @@ -231,168 +297,276 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleExp(HloInstruction* exp, HloInstruction* operand) override { + Status HandleExp(HloInstruction* exp) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[exp], ElementWiseUnaryOp(exp, [](ReturnT elem_operand) { return std::exp(elem_operand); })); return Status::OK(); - }; + } - Status HandleFloor(HloInstruction* floor, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleFloor(HloInstruction* floor) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[floor], ElementWiseUnaryOp(floor, [](ReturnT elem_operand) { return std::floor(elem_operand); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleFloor(HloInstruction* floor) { + return InvalidArgument("Unsupported type for Floor"); + } + + Status HandleFloor(HloInstruction* floor) override { + return HandleFloor(floor); + } - Status HandleLog(HloInstruction* log, HloInstruction* operand) override { + Status HandleLog(HloInstruction* log) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[log], ElementWiseUnaryOp(log, [](ReturnT elem_operand) { return std::log(elem_operand); })); return Status::OK(); - }; + } - Status HandleNot(HloInstruction* not_, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleNot(HloInstruction* not_) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[not_], ElementWiseUnaryOp(not_, [](ReturnT elem_operand) { return !elem_operand; })); return Status::OK(); - }; + } - Status HandleNegate(HloInstruction* negate, - HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleNot(HloInstruction* not_) { + return InvalidArgument("Unsupported type for Not"); + } + + Status HandleNot(HloInstruction* not_) override { + return HandleNot(not_); + } + + Status HandleNegate(HloInstruction* negate) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[negate], ElementWiseUnaryOp(negate, [](ReturnT elem_operand) { return -elem_operand; })); return Status::OK(); - }; + } - Status HandleSign(HloInstruction* sign, HloInstruction* operand) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleSign(HloInstruction* sign) { TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign], ElementWiseUnaryOp(sign, [](ReturnT elem_operand) { return (ReturnT(0) < elem_operand) - (elem_operand < ReturnT(0)); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleSign(HloInstruction* sign) { + TF_ASSIGN_OR_RETURN(parent_->evaluated_[sign], + ElementWiseUnaryOp(sign, [](ReturnT elem_operand) { + auto abs_val = std::abs(elem_operand); + return 0 == abs_val ? ReturnT(0) + : elem_operand / abs_val; + })); + return Status::OK(); + } + + Status HandleSign(HloInstruction* sign) override { + return HandleSign(sign); + } - Status HandleTanh(HloInstruction* tanh, HloInstruction* operand) override { + Status HandleTanh(HloInstruction* tanh) override { TF_ASSIGN_OR_RETURN(parent_->evaluated_[tanh], ElementWiseUnaryOp(tanh, [](ReturnT elem_operand) { return std::tanh(elem_operand); })); return Status::OK(); - }; + } - Status HandleMultiply(HloInstruction* multiply, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleMultiply(HloInstruction* multiply) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[multiply], ElementWiseBinaryOp(multiply, [](ReturnT lhs_elem, ReturnT rhs_elem) { return lhs_elem * rhs_elem; })); return Status::OK(); - }; + } - Status HandleSubtract(HloInstruction* subtract, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleSubtract(HloInstruction* subtract) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[subtract], ElementWiseBinaryOp(subtract, [](ReturnT lhs_elem, ReturnT rhs_elem) { return lhs_elem - rhs_elem; })); return Status::OK(); - }; + } - Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleAdd(HloInstruction* add) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[add], ElementWiseBinaryOp(add, [](ReturnT lhs_elem, ReturnT rhs_elem) { return lhs_elem + rhs_elem; })); return Status::OK(); - }; + } - Status HandleDivide(HloInstruction* divide, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleDivide(HloInstruction* divide) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[divide], ElementWiseBinaryOp(divide, [](ReturnT lhs_elem, ReturnT rhs_elem) { return lhs_elem / rhs_elem; })); return Status::OK(); - }; + } - Status HandleMaximum(HloInstruction* maximum) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMaximum(HloInstruction* maximum) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[maximum], ElementWiseBinaryOp(maximum, [](ReturnT lhs, ReturnT rhs) { return std::fmax(lhs, rhs); })); return Status::OK(); - }; + } - Status HandleMinimum(HloInstruction* minimum) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMaximum(HloInstruction* maximum) { + return InvalidArgument("Unsupported type for Maximum"); + } + + Status HandleMaximum(HloInstruction* maximum) override { + return HandleMaximum(maximum); + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMinimum(HloInstruction* minimum) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[minimum], ElementWiseBinaryOp(minimum, [](ReturnT lhs_el, ReturnT rhs_el) { return std::fmin(lhs_el, rhs_el); })); return Status::OK(); - }; + } - Status HandlePower(HloInstruction* power, HloInstruction* lhs, - HloInstruction* rhs) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleMinimum(HloInstruction* minimum) { + return InvalidArgument("Unsupported type for Minimum"); + } + + Status HandleMinimum(HloInstruction* minimum) override { + return HandleMinimum(minimum); + } + + Status HandlePower(HloInstruction* power) override { TF_ASSIGN_OR_RETURN( parent_->evaluated_[power], ElementWiseBinaryOp(power, [](ReturnT lhs_el, ReturnT rhs_el) { return std::pow(lhs_el, rhs_el); })); return Status::OK(); - }; + } - Status HandleRemainder(HloInstruction* remainder, HloInstruction* lhs, - HloInstruction* rhs) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRemainder(HloInstruction* remainder) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[remainder], ElementWiseBinaryOp(remainder, [](ReturnT lhs_el, ReturnT rhs_el) { return std::fmod(lhs_el, rhs_el); })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleRemainder(HloInstruction* remainder) { + return InvalidArgument("Unsupported type for Remainder"); + } - Status HandleAnd(HloInstruction* and_, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleRemainder(HloInstruction* remainder) override { + return HandleRemainder(remainder); + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleAnd(HloInstruction* and_) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[and_], ElementWiseBinaryOp(and_, [](ReturnT lhs_el, ReturnT rhs_el) { return lhs_el && rhs_el; })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleAnd(HloInstruction* and_) { + return InvalidArgument("Unsupported type for And"); + } - Status HandleOr(HloInstruction* or_, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleAnd(HloInstruction* and_) override { + return HandleAnd(and_); + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleOr(HloInstruction* or_) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[or_], ElementWiseBinaryOp(or_, [](ReturnT lhs_el, ReturnT rhs_el) { return lhs_el || rhs_el; })); return Status::OK(); - }; + } + + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleOr(HloInstruction* or_) { + return InvalidArgument("Unsupported type for Or"); + } + + Status HandleOr(HloInstruction* or_) override { + return HandleOr(or_); + } template ::value && !std::is_same::value>::type* = nullptr> - Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftLeft(HloInstruction* shl) { TF_ASSIGN_OR_RETURN( parent_->evaluated_[shl], ElementWiseBinaryOp(shl, [](NativeT lhs_elem, NativeT rhs_elem) { @@ -405,21 +579,18 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if::value || std::is_same::value>::type* = nullptr> - Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftLeft(HloInstruction*) { return InvalidArgument("Unsupported type for ShiftLeft"); } - Status HandleShiftLeft(HloInstruction* shl, HloInstruction* lhs, - HloInstruction* rhs) override { - return HandleShiftLeft(shl, lhs, rhs); + Status HandleShiftLeft(HloInstruction* shl) override { + return HandleShiftLeft(shl); } template ::value && !std::is_same::value>::type* = nullptr> - Status HandleShiftRightArithmetic(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightArithmetic(HloInstruction* shr) { typedef typename std::make_signed::type SignedT; TF_ASSIGN_OR_RETURN( parent_->evaluated_[shr], @@ -434,22 +605,19 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if::value || std::is_same::value>::type* = nullptr> - Status HandleShiftRightArithmetic(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightArithmetic(HloInstruction*) { return InvalidArgument("Unsupported type for ShiftRightArithmetic"); } - Status HandleShiftRightArithmetic(HloInstruction* shra, HloInstruction* lhs, - HloInstruction* rhs) override { - return HandleShiftRightArithmetic(shra, lhs, rhs); + Status HandleShiftRightArithmetic(HloInstruction* shra) override { + return HandleShiftRightArithmetic(shra); } template ::value && !std::is_same::value>::type* = nullptr> - Status HandleShiftRightLogical(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightLogical(HloInstruction* shr) { typedef typename std::make_unsigned::type UnsignedT; TF_ASSIGN_OR_RETURN( parent_->evaluated_[shr], @@ -464,18 +632,18 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { typename std::enable_if::value || std::is_same::value>::type* = nullptr> - Status HandleShiftRightLogical(HloInstruction* shr, HloInstruction* lhs, - HloInstruction* rhs) { + Status HandleShiftRightLogical(HloInstruction*) { return InvalidArgument("Unsupported type for ShiftRightLogical"); } - Status HandleShiftRightLogical(HloInstruction* shrl, HloInstruction* lhs, - HloInstruction* rhs) override { - return HandleShiftRightLogical(shrl, lhs, rhs); + Status HandleShiftRightLogical(HloInstruction* shrl) override { + return HandleShiftRightLogical(shrl); } - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleClamp(HloInstruction* clamp) { std::function clamp_op = [](ReturnT low, ReturnT high, ReturnT value) { return std::fmax(low, std::fmin(value, high)); @@ -483,11 +651,20 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { TF_ASSIGN_OR_RETURN(parent_->evaluated_[clamp], ElementWiseTernaryOp(clamp, std::move(clamp_op))); return Status::OK(); - }; + } - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override { + template < + typename NativeT, + typename std::enable_if::value>::type* = nullptr> + Status HandleClamp(HloInstruction*) { + return InvalidArgument("Unsupported type for Clamp"); + } + + Status HandleClamp(HloInstruction* clamp) override { + return HandleClamp(clamp); + } + + Status HandleSelect(HloInstruction* select) override { CHECK(!ShapeUtil::IsTuple(select->shape())); std::function select_op = [](bool pred, ReturnT on_true, ReturnT on_false) { @@ -499,13 +676,13 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { TF_ASSIGN_OR_RETURN(parent_->evaluated_[select], ElementWiseTernaryOp(select, std::move(select_op))); return Status::OK(); - }; + } - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override { + Status HandleReverse(HloInstruction* reverse) override { const auto result_shape = reverse->shape(); const auto reverse_dimensions = reverse->dimensions(); + auto operand = reverse->operand(0); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, ShapeInference::InferReverseShape(operand->shape(), reverse_dimensions)); @@ -529,10 +706,12 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[reverse] = std::move(result); return Status::OK(); - }; + } - Status HandleConvolution(HloInstruction* conv, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override { + Status HandleConvolution(HloInstruction* conv) override { + auto lhs = conv->operand(0); + auto rhs = conv->operand(1); + const auto& window = conv->window(); const Shape& result_shape = conv->shape(); const Shape& lhs_shape = lhs->shape(); const Shape& rhs_shape = rhs->shape(); @@ -547,7 +726,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { const auto& dnums = conv->convolution_dimension_numbers(); const int64 num_spatial_dims = dnums.spatial_dimensions_size(); CHECK_EQ(num_spatial_dims, dnums.kernel_spatial_dimensions_size()); - CHECK_GE(num_spatial_dims, 1); + CHECK_GE(num_spatial_dims, 0); CHECK_EQ(window.dimensions_size(), num_spatial_dims); const auto lhs_rank = ShapeUtil::Rank(lhs_shape); @@ -652,10 +831,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[conv] = std::move(result); return Status::OK(); - }; + } - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleDot(HloInstruction* dot) override { + auto lhs = dot->operand(0); + auto rhs = dot->operand(1); CHECK(ShapeUtil::IsArray(dot->shape())); CHECK(ShapeUtil::IsArray(lhs->shape())); CHECK(ShapeUtil::IsArray(rhs->shape())); @@ -719,7 +899,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[dot] = std::move(result); return Status::OK(); - }; + } Status HandlePad(HloInstruction* pad) override { CHECK(!ShapeUtil::IsTuple(pad->operand(0)->shape())); @@ -788,11 +968,11 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[pad] = std::move(result); return Status::OK(); - }; + } - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) override { + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override { + auto operand = dynamic_slice->operand(0); + auto start_indices = dynamic_slice->operand(1); auto result_shape = dynamic_slice->shape(); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, ShapeInference::InferDynamicSliceShape( @@ -841,12 +1021,13 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } return Status::OK(); - }; + } - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override { + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override { + auto operand = dynamic_update_slice->operand(0); + auto update = dynamic_update_slice->operand(1); + auto start_indices = dynamic_update_slice->operand(2); auto result_shape = dynamic_update_slice->shape(); TF_ASSIGN_OR_RETURN( auto inferred_return_shape, @@ -897,12 +1078,13 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } return Status::OK(); - }; + } - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override { + Status HandleReduce(HloInstruction* reduce) override { + auto arg = reduce->operand(0); + auto init_value = reduce->operand(1); + tensorflow::gtl::ArraySlice dimensions(reduce->dimensions()); + HloComputation* function = reduce->to_apply(); TF_RET_CHECK(ShapeUtil::Rank(reduce->shape()) == ShapeUtil::Rank(arg->shape()) - dimensions.size()); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, @@ -985,11 +1167,12 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[reduce] = std::move(result); return Status::OK(); - }; + } - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override { + Status HandleReduceWindow(HloInstruction* reduce_window) override { + auto operand = reduce_window->operand(0); + const Window& window = reduce_window->window(); + HloComputation* function = reduce_window->to_apply(); TF_ASSIGN_OR_RETURN( auto inferred_return_shape, ShapeInference::InferReduceWindowShape( @@ -1072,9 +1255,10 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { parent_->evaluated_[reduce_window] = std::move(result); return Status::OK(); - }; + } - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override { + Status HandleSlice(HloInstruction* slice) override { + auto operand = slice->operand(0); const Shape& shape = slice->shape(); TF_ASSIGN_OR_RETURN(auto inferred_return_shape, ShapeInference::InferSliceShape( @@ -1101,7 +1285,7 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { TF_RETURN_IF_ERROR(result->Populate(func)); parent_->evaluated_[slice] = std::move(result); return Status::OK(); - }; + } private: template @@ -1244,32 +1428,33 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault { } HloEvaluator* parent_; -}; // namespace xla +}; // class HloEvaluator::TypedVisitor HloEvaluator::HloEvaluator() { typed_visitors_[PRED] = MakeUnique>(this); typed_visitors_[U8] = MakeUnique>(this); typed_visitors_[U16] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: U16."); + return Unimplemented("HloEvaluator: unhandled primitive type: U16."); }); typed_visitors_[U32] = MakeUnique>(this); typed_visitors_[U64] = MakeUnique>(this); typed_visitors_[S8] = MakeUnique>(this); typed_visitors_[S16] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: S16."); + return Unimplemented("HloEvaluator: unhandled primitive type: S16."); }); typed_visitors_[S32] = MakeUnique>(this); typed_visitors_[S64] = MakeUnique>(this); typed_visitors_[F16] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: F16."); + return Unimplemented("HloEvaluator: unhandled primitive type: F16."); }); typed_visitors_[F32] = MakeUnique>(this); typed_visitors_[F64] = MakeUnique>(this); + typed_visitors_[C64] = MakeUnique>(this); typed_visitors_[TUPLE] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: TUPLE."); + return Unimplemented("HloEvaluator: unhandled primitive type: TUPLE."); }); typed_visitors_[OPAQUE] = MakeUnique([](HloInstruction*) { - return Unimplemented("unhandled primitive type: OPAQUE."); + return Unimplemented("HloEvaluator: unhandled primitive type: OPAQUE."); }); } @@ -1402,10 +1587,7 @@ Status HloEvaluator::HandleParameter(HloInstruction* parameter) { return Status::OK(); } -Status HloEvaluator::HandleConstant(HloInstruction* constant, - const Literal& literal) { - return Status::OK(); -} +Status HloEvaluator::HandleConstant(HloInstruction*) { return Status::OK(); } Status HloEvaluator::HandleReshape(HloInstruction* reshape) { TF_ASSIGN_OR_RETURN( @@ -1421,9 +1603,9 @@ Status HloEvaluator::HandleTranspose(HloInstruction* transpose) { return Status::OK(); } -Status HloEvaluator::HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) { +Status HloEvaluator::HandleConcatenate(HloInstruction* concatenate) { + tensorflow::gtl::ArraySlice operands( + concatenate->operands()); // The result concatenate dimension is going to be the sum of all concatenate // dimensions of the operands taking part of the operation. const Shape& reference_shape = operands[0]->shape(); @@ -1463,8 +1645,8 @@ Status HloEvaluator::HandleConcatenate( return Status::OK(); } -Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite, - HloInstruction* operand) { +Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite) { + auto operand = is_finite->operand(0); if (!ShapeUtil::ElementIsFloating(operand->shape())) { return InvalidArgument( "expected element type in shape to be float for IsFinite op, got: %s", @@ -1498,8 +1680,10 @@ Status HloEvaluator::HandleIsFinite(HloInstruction* is_finite, return Status::OK(); } -Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) { +Status HloEvaluator::HandleCompare(HloInstruction* compare) { + HloOpcode opcode = compare->opcode(); + auto lhs = compare->operand(0); + auto rhs = compare->operand(1); // TODO(b/35950897, b/27796129): add DCHECK back once implicit broadcast is // removed. if (!(ShapeUtil::SameDimensions(compare->shape(), rhs->shape()) && @@ -1570,6 +1754,11 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode, evaluated_[compare], Compare(compare->shape(), opcode, lhs_literal, rhs_literal)); } break; + case C64: { + TF_ASSIGN_OR_RETURN(evaluated_[compare], + Compare(compare->shape(), opcode, + lhs_literal, rhs_literal)); + } break; default: LOG(FATAL) << "HandleCompare: unknown primitive type: " << PrimitiveType_Name(lhs->shape().element_type()); @@ -1578,11 +1767,9 @@ Status HloEvaluator::HandleCompare(HloInstruction* compare, HloOpcode opcode, return Status::OK(); } -Status HloEvaluator::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status HloEvaluator::HandleTuple(HloInstruction* tuple) { std::vector operand_literals; - for (auto operand : operands) { + for (auto operand : tuple->operands()) { operand_literals.push_back(&GetEvaluatedLiteralFor(operand)); } @@ -1590,11 +1777,11 @@ Status HloEvaluator::HandleTuple( return Status::OK(); } -Status HloEvaluator::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status HloEvaluator::HandleGetTupleElement(HloInstruction* get_tuple_element) { const auto result_shape = get_tuple_element->shape(); const int64 index = get_tuple_element->tuple_index(); + auto operand = get_tuple_element->operand(0); TF_ASSIGN_OR_RETURN( auto inferred_return_shape, ShapeInference::InferGetTupleElementShape(operand->shape(), index)); diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.h b/tensorflow/compiler/xla/service/hlo_evaluator.h index a9cecb11be3211a645b567b44f6c05c419b7835b..67b6e215fcb23598f1a8ab6212d6e7e58a64e976 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.h +++ b/tensorflow/compiler/xla/service/hlo_evaluator.h @@ -120,28 +120,20 @@ class HloEvaluator : public DfsHloVisitorWithDefault { // Status HandleParameter(HloInstruction* parameter) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override; + Status HandleConcatenate(HloInstruction* concatenate) override; Status HandleReshape(HloInstruction* reshape) override; Status HandleTranspose(HloInstruction* transpose) override; - Status HandleIsFinite(HloInstruction* is_finite, - HloInstruction* operand) override; + Status HandleIsFinite(HloInstruction* is_finite) override; - Status HandleCompare(HloInstruction* compare, HloOpcode opcode, - HloInstruction* lhs, HloInstruction* rhs) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; + Status HandleCompare(HloInstruction* compare) override; + Status HandleTuple(HloInstruction* tuple) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleCopy(HloInstruction* copy) override; diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc index 5172739624861972a32802a5148032eb83f6cda6..85477af6fe26f53504c07204348566c16a24392c 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc @@ -30,7 +30,7 @@ limitations under the License. #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/test.h" -#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/util.h" @@ -41,7 +41,7 @@ limitations under the License. namespace xla { namespace { -class HloEvaluatorTest : public HloTestBase { +class HloEvaluatorTest : public HloVerifiedTestBase { protected: HloEvaluatorTest() { evaluator_ = MakeUnique(); } @@ -62,8 +62,7 @@ TEST_F(HloEvaluatorTest, DoesClamp) { auto c3 = b.AddInstruction(HloInstruction::CreateConstant(std::move(value))); auto instruction = b.AddInstruction( HloInstruction::CreateTernary(shape, HloOpcode::kClamp, c1, c2, c3)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); @@ -89,8 +88,7 @@ TEST_F(HloEvaluatorTest, DoesSelect) { b.AddInstruction(HloInstruction::CreateConstant(std::move(on_false))); auto instruction = b.AddInstruction( HloInstruction::CreateTernary(shape, HloOpcode::kSelect, c1, c2, c3)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); @@ -112,8 +110,7 @@ TEST_F(HloEvaluatorTest, DoesAdd) { auto c2 = b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs))); auto instruction = b.AddInstruction( HloInstruction::CreateBinary(shape, HloOpcode::kAdd, c1, c2)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); @@ -125,111 +122,100 @@ TEST_F(HloEvaluatorTest, DoesAdd) { // Verifies that HloEvaluator evaluates a HLO instruction that performs // element-wise divide with 2 operands. -TEST_F(HloEvaluatorTest, DoesDivide) { - { - auto lhs_s64 = Literal::CreateR2({{1, 0}, {-100, 4}}); - auto rhs_s64 = Literal::CreateR2({{2, 4}, {4, 4}}); - - Shape shape_s64 = ShapeUtil::MakeShape(S64, {2, 2}); - HloComputation::Builder b(TestName()); - auto c1_s64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_s64))); - auto c2_s64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_s64))); - auto instruction = b.AddInstruction(HloInstruction::CreateBinary( - shape_s64, HloOpcode::kDivide, c1_s64, c2_s64)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - std::unique_ptr result = - evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); - - auto expected = Literal::CreateR2({{0, 0}, {-25, 1}}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } - { - auto lhs_f64 = Literal::CreateR2({{1.0, 0.0}, {-100.0, 4.0}}); - auto rhs_f64 = Literal::CreateR2({{2.2, 4.0}, {4.0, 4.0}}); - - Shape shape_f64 = ShapeUtil::MakeShape(F64, {2, 2}); - HloComputation::Builder b(TestName()); - auto c1_f64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_f64))); - auto c2_f64 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_f64))); - auto instruction = b.AddInstruction(HloInstruction::CreateBinary( - shape_f64, HloOpcode::kDivide, c1_f64, c2_f64)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - auto result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); - - auto expected = - Literal::CreateR2({{0.45454545454545453, 0}, {-25, 1}}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } +TEST_F(HloEvaluatorTest, DoesDivideInt64) { + auto lhs_s64 = Literal::CreateR2({{1, 0}, {-100, 4}}); + auto rhs_s64 = Literal::CreateR2({{2, 4}, {4, 4}}); + + Shape shape_s64 = ShapeUtil::MakeShape(S64, {2, 2}); + HloComputation::Builder b(TestName()); + auto c1_s64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_s64))); + auto c2_s64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_s64))); + auto instruction = b.AddInstruction(HloInstruction::CreateBinary( + shape_s64, HloOpcode::kDivide, c1_s64, c2_s64)); + module().AddEntryComputation(b.Build()); + + std::unique_ptr result = + evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); + + auto expected = Literal::CreateR2({{0, 0}, {-25, 1}}); + + LiteralTestUtil::ExpectEqual(*expected, *result); +} +TEST_F(HloEvaluatorTest, DoesDivideDouble) { + auto lhs_f64 = Literal::CreateR2({{1.0, 0.0}, {-100.0, 4.0}}); + auto rhs_f64 = Literal::CreateR2({{2.2, 4.0}, {4.0, 4.0}}); + + Shape shape_f64 = ShapeUtil::MakeShape(F64, {2, 2}); + HloComputation::Builder b(TestName()); + auto c1_f64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(lhs_f64))); + auto c2_f64 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(rhs_f64))); + auto instruction = b.AddInstruction(HloInstruction::CreateBinary( + shape_f64, HloOpcode::kDivide, c1_f64, c2_f64)); + module().AddEntryComputation(b.Build()); + + auto result = evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); + + auto expected = + Literal::CreateR2({{0.45454545454545453, 0}, {-25, 1}}); + + LiteralTestUtil::ExpectEqual(*expected, *result); } // Verifies that HloEvaluator evaluates a HLO instruction that performs // element-wise abs op with 1 operand. -TEST_F(HloEvaluatorTest, DoesAbs) { - { - auto operand = Literal::CreateR2({{1, -20}, {-100, 4}}); - const Shape& shape = ShapeUtil::MakeShape(S64, {2, 2}); - HloComputation::Builder b(TestName()); - auto c1 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); - auto instruction = b.AddInstruction( - HloInstruction::CreateUnary(shape, HloOpcode::kAbs, c1)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - std::unique_ptr result = - evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); - - auto expected = Literal::CreateR2({{1, 20}, {100, 4}}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } +TEST_F(HloEvaluatorTest, DoesAbsR2) { + auto operand = Literal::CreateR2({{1, -20}, {-100, 4}}); + const Shape& shape = ShapeUtil::MakeShape(S64, {2, 2}); + HloComputation::Builder b(TestName()); + auto c1 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); + auto instruction = + b.AddInstruction(HloInstruction::CreateUnary(shape, HloOpcode::kAbs, c1)); + module().AddEntryComputation(b.Build()); + std::unique_ptr result = + evaluator_->Evaluate(instruction, {}).ConsumeValueOrDie(); + + auto expected = Literal::CreateR2({{1, 20}, {100, 4}}); + + LiteralTestUtil::ExpectEqual(*expected, *result); +} +TEST_F(HloEvaluatorTest, DoesAbsR0) { // For R0 literal. - { - const Shape& r0 = ShapeUtil::MakeShape(F32, {}); - auto operand = Literal::CreateR0(-1.0f); - HloComputation::Builder b(TestName()); - auto c1 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); - auto instruction = - b.AddInstruction(HloInstruction::CreateUnary(r0, HloOpcode::kAbs, c1)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); - auto expected = Literal::CreateR0(1.0f); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } + const Shape& r0 = ShapeUtil::MakeShape(F32, {}); + auto operand = Literal::CreateR0(-1.0f); + HloComputation::Builder b(TestName()); + auto c1 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); + auto instruction = + b.AddInstruction(HloInstruction::CreateUnary(r0, HloOpcode::kAbs, c1)); + module().AddEntryComputation(b.Build()); + + auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); + auto expected = Literal::CreateR0(1.0f); + LiteralTestUtil::ExpectEqual(*expected, *result); +} +TEST_F(HloEvaluatorTest, DoesAbsR1WithZeroSize) { // For R1 literal with dimension of size 0. - { - Shape empty_r1 = ShapeUtil::MakeShape(F32, {0}); - auto operand = Literal::CreateR1({}); - HloComputation::Builder b(TestName()); - auto c1 = - b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); - auto instruction = b.AddInstruction( - HloInstruction::CreateUnary(empty_r1, HloOpcode::kAbs, c1)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); - - auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); - auto expected = Literal::CreateR1({}); - - LiteralTestUtil::ExpectEqual(*expected, *result); - } -} // namespace + Shape empty_r1 = ShapeUtil::MakeShape(F32, {0}); + auto operand = Literal::CreateR1({}); + HloComputation::Builder b(TestName()); + auto c1 = + b.AddInstruction(HloInstruction::CreateConstant(std::move(operand))); + auto instruction = b.AddInstruction( + HloInstruction::CreateUnary(empty_r1, HloOpcode::kAbs, c1)); + module().AddEntryComputation(b.Build()); + + auto result = evaluator_->Evaluate(instruction).ConsumeValueOrDie(); + auto expected = Literal::CreateR1({}); + + LiteralTestUtil::ExpectEqual(*expected, *result); +} // Verifies that HloEvaluator evaluates a HLO Computation with non-parameter nor // constant operands. @@ -253,8 +239,7 @@ TEST_F(HloEvaluatorTest, DoesTraverseInstructions) { b.AddInstruction(HloInstruction::CreateParameter(2, shape, "rhs2")); b.AddInstruction(HloInstruction::CreateBinary(shape, HloOpcode::kAdd, lhs_instruction, param_rhs2)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, args).ConsumeValueOrDie(); @@ -279,8 +264,7 @@ TEST_F(HloEvaluatorTest, DoesReshape) { const int64 permutation[] = {1, 2, 0, 4, 3}; b.AddInstruction( HloInstruction::CreateTranspose(shape, literal_instruction, permutation)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -303,8 +287,7 @@ TEST_F(HloEvaluatorTest, DoesBroadcast) { HloInstruction::CreateConstant(std::move(input_literal))); b.AddInstruction(HloInstruction::CreateBroadcast( output_literal->shape(), literal_instruction, {1, 2})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -324,8 +307,7 @@ TEST_F(HloEvaluatorTest, DoesBroadcastScalar) { b.AddInstruction(HloInstruction::CreateBroadcast( output_literal->shape(), literal_instruction, /*broadcast_dimensions=*/{})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -343,11 +325,10 @@ TEST_F(HloEvaluatorTest, DoesConcatenateSimple) { std::vector operands = {operand1, operand2}; - Shape shape = ShapeUtil::MakeShape(S64, {2, 2}); + Shape shape = ShapeUtil::MakeShape(S64, {4, 2}); b.AddInstruction(HloInstruction::CreateConcatenate(shape, operands, 0)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -370,8 +351,7 @@ TEST_F(HloEvaluatorTest, ConcatenateHandlesShapeWithZeroElement) { Shape shape = ShapeUtil::MakeShape(S64, {2}); b.AddInstruction(HloInstruction::CreateConcatenate(shape, operands, 0)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -392,8 +372,7 @@ TEST_F(HloEvaluatorTest, ConvertWithSameLayout) { HloInstruction* constant = b.AddInstruction( HloInstruction::CreateConstant(std::move(input_literal))); b.AddInstruction(HloInstruction::CreateConvert(expected->shape(), constant)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -414,8 +393,7 @@ TEST_F(HloEvaluatorTest, ConvertWithDifferentLayout) { HloInstruction* constant = b.AddInstruction( HloInstruction::CreateConstant(std::move(input_literal))); b.AddInstruction(HloInstruction::CreateConvert(expected->shape(), constant)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -451,8 +429,7 @@ TEST_F(HloEvaluatorTest, Pad2DIntegerArrayWithZeroDimension) { Shape shape = ShapeUtil::MakeShape(S32, {5, 2}); auto pad_instruction = b.AddInstruction(HloInstruction::CreatePad( shape, operand_instruction, padding_value_instruction, padding_config)); - HloModule module(TestName()); - module.AddEntryComputation(b.Build()); + module().AddEntryComputation(b.Build()); auto result = evaluator_->Evaluate(pad_instruction).ConsumeValueOrDie(); @@ -479,8 +456,7 @@ TEST_F(HloEvaluatorTest, Pad4DFloatArrayWithInteriorPadding) { CreatePaddingConfig({{{1, 0, 2}}, {{0, 2, 1}}, {{0, 0, 0}}, {{0, 0, 0}}}); b.AddInstruction(HloInstruction::CreatePad( shape, input_instruction, pad_instruction, r4_padding_on_dim0_dim1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -525,8 +501,7 @@ TEST_F(HloEvaluatorTest, NegativePadding2D) { pad_value_instruction, r2_padding_on_dim0_dim1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -572,8 +547,7 @@ TEST_F(HloEvaluatorTest, NegativeAndInteriorPadding2D) { pad_value_instruction, r2_padding_on_dim0_dim1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -609,8 +583,7 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank1) { Shape shape = ShapeUtil::MakeShape(F32, {4, 2}); b.AddInstruction(HloInstruction::CreateBinary( shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -653,8 +626,7 @@ TEST_F(HloEvaluatorTest, DotRank1AndRank2) { Shape shape = ShapeUtil::MakeShape(F32, {2}); b.AddInstruction(HloInstruction::CreateBinary( shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -695,8 +667,7 @@ TEST_F(HloEvaluatorTest, DotRank2AndRank2) { Shape shape = ShapeUtil::MakeShape(F32, {4, 2}); b.AddInstruction(HloInstruction::CreateBinary( shape, HloOpcode::kDot, lhs_instruction, rhs_instruction)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -749,8 +720,7 @@ TEST_F(HloEvaluatorTest, SimpleConv1D) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 3}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -805,8 +775,7 @@ TEST_F(HloEvaluatorTest, Simple4x4Conv2DWith2x2Kernel) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 4, 4}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -885,8 +854,7 @@ TEST_F(HloEvaluatorTest, Conv2DGeneralDimensions) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 1, 2}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -944,8 +912,7 @@ TEST_F(HloEvaluatorTest, DilatedBaseConv2DWithHighPadding) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 7, 7}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1009,8 +976,7 @@ TEST_F(HloEvaluatorTest, DilatedBaseConv2DWithLowAndHighPadding) { const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 8, 8}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1082,8 +1048,7 @@ TEST_F(HloEvaluatorTest, const Shape& shape = ShapeUtil::MakeShape(F32, {1, 1, 9, 3}); b.AddInstruction(HloInstruction::CreateConvolve( shape, lhs_instruction, rhs_instruction, window, dnums)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1131,15 +1096,14 @@ TEST_F(HloEvaluatorTest, ReduceAdd) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); add_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs)); - HloModule module(TestName()); - auto add_func = module.AddEmbeddedComputation(add_computation.Build()); + auto add_func = module().AddEmbeddedComputation(add_computation.Build()); Shape shape = ShapeUtil::MakeShape(F32, {2}); b.AddInstruction( HloInstruction::CreateReduce(shape, arg_instruction, init_value, /*dimensions_to_reduce=*/{1}, add_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1175,8 +1139,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowMax) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); max_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kMaximum, param_lhs, param_rhs)); - HloModule module(TestName()); - auto max_func = module.AddEmbeddedComputation(max_computation.Build()); + auto max_func = module().AddEmbeddedComputation(max_computation.Build()); Window window; WindowDimension dim; @@ -1193,7 +1156,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowMax) { b.AddInstruction(HloInstruction::CreateReduceWindow( shape, arg_instruction, init_value, window, max_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1227,8 +1190,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); add_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs)); - HloModule module(TestName()); - auto add_func = module.AddEmbeddedComputation(add_computation.Build()); + auto add_func = module().AddEmbeddedComputation(add_computation.Build()); Window window; WindowDimension dim; @@ -1251,7 +1213,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd) { b.AddInstruction(HloInstruction::CreateReduceWindow( shape, arg_instruction, init_value, window, add_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1281,8 +1243,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd6D) { HloInstruction::CreateParameter(1, scalar_shape, "rhs")); add_computation.AddInstruction(HloInstruction::CreateBinary( scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs)); - HloModule module(TestName()); - auto add_func = module.AddEmbeddedComputation(add_computation.Build()); + auto add_func = module().AddEmbeddedComputation(add_computation.Build()); Window window; @@ -1313,7 +1274,7 @@ TEST_F(HloEvaluatorTest, ReduceWindowAdd6D) { b.AddInstruction(HloInstruction::CreateReduceWindow( shape, arg_instruction, init_value, window, add_func)); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1344,8 +1305,7 @@ TEST_F(HloEvaluatorTest, StridedSlice) { /*start_indices=*/{0, 2}, /*limit_indices=*/{3, 5}, /*strides=*/{2, 3})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1379,8 +1339,7 @@ TEST_F(HloEvaluatorTest, DynamicSlice) { Shape shape = ShapeUtil::MakeShape(F32, {2, 3}); b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand, start_indices, {2, 3})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1416,8 +1375,7 @@ TEST_F(HloEvaluatorTest, DynamicSliceModSlice) { Shape shape = ShapeUtil::MakeShape(F32, {2, 3}); b.AddInstruction(HloInstruction::CreateDynamicSlice(shape, operand, start_indices, {2, 3})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1454,8 +1412,7 @@ TEST_F(HloEvaluatorTest, DynamicSliceUpdate) { Shape shape = ShapeUtil::MakeShape(F64, {2, 3}); b.AddInstruction(HloInstruction::CreateDynamicUpdateSlice( shape, operand, update, start_indices)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1491,8 +1448,7 @@ TEST_F(HloEvaluatorTest, SetAndGetTuples) { Shape shape = ShapeUtil::MakeShape(F64, {2, 3}); b.AddInstruction(HloInstruction::CreateGetTupleElement(shape, tuple, 1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1531,8 +1487,7 @@ TEST_F(HloEvaluatorTest, SetAndGetNestedTuples) { b.AddInstruction( HloInstruction::CreateGetTupleElement(tuple2->shape(), outer_tuple, 1)); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); @@ -1572,8 +1527,7 @@ TEST_F(HloEvaluatorTest, Reverse) { const Shape shape = ShapeUtil::MakeShape(F32, {4, 3, 2, 1}); b.AddInstruction(HloInstruction::CreateReverse(shape, operand, {0, 1})); - HloModule module(TestName()); - auto computation = module.AddEntryComputation(b.Build()); + auto computation = module().AddEntryComputation(b.Build()); std::unique_ptr result = evaluator_->Evaluate(*computation, {}).ConsumeValueOrDie(); diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc index 24e390529e5cd02a4bb40d7aa861e852254fe253..d7bdd4117d947add448ff660abc621d9ae3118b6 100644 --- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc +++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc @@ -231,9 +231,9 @@ string HtmlLikeStringSanitize(tensorflow::StringPiece s) { // commutative, we also support them with param0 and param1 swapped. // // This is useful primarily for reduce and map nodes. These take a -// subcomputation which is almost always one of the four above, and pattern -// matching it to a short string lets us tell the user what the subcomputation -// is without drawing it as a graph. +// subcomputation which is almost always one of the above, and pattern matching +// it to a short string lets us tell the user what the subcomputation is without +// drawing it as a graph. optional MatchTrivialComputation(const HloComputation* computation) { if (computation->instruction_count() != 3) { return nullopt; @@ -342,6 +342,11 @@ class HloDotDumper { bool ShouldShowSubcomputation(const HloComputation* subcomp); bool ShouldShowFusionSubcomputation(const HloInstruction* instr); + + // We omit some nodes from the graph, instead drawing them inlined into the + // nodes that use them. + bool ShouldMergeIntoUsers(const HloInstruction* instr) const; + string DumpSubcomputation(const HloComputation* subcomp, const HloInstruction* parent_instr); string DumpComputation(const HloComputation* comp); @@ -352,9 +357,24 @@ class HloDotDumper { string GetInstructionNodeLabel(const HloInstruction* instr); string GetInstructionNodeMetadata(const HloInstruction* instr); string GetInstructionNodeExtraInfo(const HloInstruction* instr); - string GetInstructionNodeInlinedConstants(const HloInstruction* instr); + string GetInstructionNodeInlinedOperands(const HloInstruction* instr); void AddInstructionIncomingEdges(const HloInstruction* instr); + // For most instructions, GetNodeForEdge(instr) returns instr. + // + // The exception is fusion nodes. For these, we walk up the chain of nested + // fusion nodes starting at instr until we reach a node that either (a) isn't + // a fusion node, or (b) is a fusion node for which + // ShouldShowFusionSubcomputation is false. + // + // We do this because fusion nodes are expanded inline -- if + // ShouldShowFusionSubcomputation is true, the fusion node won't be present in + // the graph. + // + // In general when you want to draw an edge from A to B, you should actually + // draw an edge from GetNodeForEdge(A) to GetNodeForEdge(B). + const HloInstruction* GetNodeForEdge(const HloInstruction* instr); + // If instr has just one computation and it's trivial (e.g. "return param0 + // param1"), returns a string you can put into the node's body that names the // subcomputation, e.g. "Subcomputation: add". @@ -590,16 +610,15 @@ tooltip = " "; // belongs to a fusion node, it's drawn in place of the fusion instruction, // so there's no need to link those. if (parent_instr->opcode() != HloOpcode::kFusion) { - VLOG(2) << "Edge: from " << subcomp->root_instruction()->name() << " to " - << parent_instr->name() << " as " << next_edge_id_; - edge_ids_.insert( - {{subcomp->root_instruction(), parent_instr}, next_edge_id_++}); + const HloInstruction* from = GetNodeForEdge(subcomp->root_instruction()); + VLOG(2) << "Edge: from " << from->name() << " to " << parent_instr->name() + << " as " << next_edge_id_; + edge_ids_.insert({{from, parent_instr}, next_edge_id_++}); const char* edge_fmt = R"(%s -> %s [ltail="%s", style="dashed" tooltip="%s -> %s"];)"; - edges_.push_back( - Printf(edge_fmt, InstructionId(subcomp->root_instruction()), - InstructionId(parent_instr), SubcomputationId(subcomp), - subcomp->name(), parent_instr->name())); + edges_.push_back(Printf( + edge_fmt, InstructionId(from), InstructionId(parent_instr), + SubcomputationId(subcomp), subcomp->name(), parent_instr->name())); } string computation = @@ -628,15 +647,7 @@ string HloDotDumper::DumpComputation(const HloComputation* comp) { } string HloDotDumper::DumpRootTag() { - HloInstruction* from = computation_->root_instruction(); - - // Fusion nodes are expanded inline, so if root is an expanded fusion node, - // walk up the graph until we find a node that isn't. - while (from->opcode() == HloOpcode::kFusion && - ShouldShowFusionSubcomputation(from)) { - from = from->fused_expression_root(); - } - + const HloInstruction* from = GetNodeForEdge(computation_->root_instruction()); auto from_id = InstructionId(from); if (!filter_.Show(from)) { @@ -668,12 +679,42 @@ string HloDotDumper::DumpRootTag() { to_id, node_body, node_shape, NodeColorAttributes(color)); } +bool HloDotDumper::ShouldMergeIntoUsers(const HloInstruction* instr) const { + // If a node: + // + // - is a tuple-shaped parameter, + // - is not a parameter to a fusion node, + // - has at least kMinUsersToOmit users shown, and + // - all of the shown users are get-tuple-elements, + // + // then we omit it from the graph, merging it with its users. + // + // This helps us handle the common case where a while loop body has one big + // tuple-shaped parameter. + const int kMinUsersToOmit = 3; + return instr->opcode() == HloOpcode::kParameter && + ShapeUtil::IsTuple(instr->shape()) && !instr->IsFused() && + std::count_if(instr->users().begin(), instr->users().end(), + [&](const HloInstruction* user) { + return filter_.Show(user); + }) > kMinUsersToOmit && + std::all_of(instr->users().begin(), instr->users().end(), + [&](const HloInstruction* user) { + return !filter_.Show(user) || + user->opcode() == HloOpcode::kGetTupleElement; + }); +} + string HloDotDumper::DumpInstruction(const HloInstruction* instr) { // We don't display constants as separate nodes; they're merged into their // users. if (instr->opcode() == HloOpcode::kConstant) { return ""; } + // Skip this node if it's merged into its users. + if (ShouldMergeIntoUsers(instr)) { + return ""; + } // Omit the fusion node if its subcomputation is drawn, since the // subcomputation will be drawn inline. if (instr->opcode() == HloOpcode::kFusion && @@ -689,7 +730,7 @@ string HloDotDumper::DumpInstruction(const HloInstruction* instr) { string node_label = GetInstructionNodeLabel(instr); string node_metadata = GetInstructionNodeMetadata(instr); string extra_info = GetInstructionNodeExtraInfo(instr); - string inlined_constants = GetInstructionNodeInlinedConstants(instr); + string inlined_constants = GetInstructionNodeInlinedOperands(instr); string trivial_subcomputation = GetInstructionTrivialComputationStr(instr); AddInstructionIncomingEdges(instr); @@ -717,7 +758,7 @@ string HloDotDumper::DumpInstruction(const HloInstruction* instr) { NodeColorAttributes(color)); } -string HloDotDumper::GetInstructionNodeInlinedConstants( +string HloDotDumper::GetInstructionNodeInlinedOperands( const HloInstruction* instr) { auto stringify_constant = [](const HloInstruction* constant) { if (ShapeUtil::IsEffectiveScalar(constant->shape())) { @@ -726,10 +767,14 @@ string HloDotDumper::GetInstructionNodeInlinedConstants( return Printf("%s (%s)", constant->literal().GetAsString(elem_idx), ShapeUtil::HumanString(constant->shape())); } + string constant_name; if (tensorflow::StringPiece(constant->name()).starts_with("%constant")) { - return constant->name(); + constant_name = constant->name(); + } else { + constant_name = StrCat("constant ", constant->name()); } - return StrCat("constant ", constant->name()); + return Printf("%s %s", constant_name, + ShapeUtil::HumanString(constant->shape())); }; // Special case: If instr is a parameter to a fusion node, check whether the @@ -746,16 +791,44 @@ string HloDotDumper::GetInstructionNodeInlinedConstants( std::vector lines; for (int64 i = 0; i < instr->operand_count(); ++i) { const HloInstruction* operand = instr->operand(i); - if (operand->opcode() != HloOpcode::kConstant) { - continue; + optional operand_str; + if (operand->opcode() == HloOpcode::kConstant) { + operand_str = stringify_constant(operand); + } else if (ShouldMergeIntoUsers(operand)) { + // Special case: If the operand is a parameter, use its parameter number + // rather than its name, because that's generally how people think of the + // node. + if (operand->opcode() == HloOpcode::kParameter) { + operand_str = Printf("Parameter %lld", operand->parameter_number()); + } else { + operand_str = operand->name(); + } + } + + if (operand_str) { + if (instr->operand_count() > 1) { + lines.push_back(Printf("operand %lld = %s", i, *operand_str)); + } else { + lines.push_back(Printf("operand = %s", *operand_str)); + } } - lines.push_back( - Printf("operand %lld = %s", i, stringify_constant(operand))); } return Join(lines, "
"); } ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { + const auto kParameterColor = kOrange; + + // Special case: If this instruction has a parameter merged into it, paint it + // the same color as a parameter. + if (std::any_of(instr->operands().begin(), instr->operands().end(), + [&](const HloInstruction* operand) { + return operand->opcode() == HloOpcode::kParameter && + ShouldMergeIntoUsers(operand); + })) { + return kParameterColor; + } + // Pick different colors or shapes for instructions which are particularly // expensive (eg, dot) and those which are unusual in some way or unique // (eg, parameter). @@ -763,8 +836,10 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kAbs: case HloOpcode::kRoundNearestAfz: case HloOpcode::kAdd: + case HloOpcode::kAtan2: case HloOpcode::kCeil: case HloOpcode::kClamp: + case HloOpcode::kComplex: case HloOpcode::kConvert: case HloOpcode::kCos: case HloOpcode::kDivide: @@ -773,7 +848,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kFloor: case HloOpcode::kGe: case HloOpcode::kGt: - case HloOpcode::kIndex: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLe: case HloOpcode::kLog: @@ -787,8 +862,8 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kNe: case HloOpcode::kNegate: case HloOpcode::kPower: + case HloOpcode::kReal: case HloOpcode::kRemainder: - case HloOpcode::kSelect: case HloOpcode::kShiftLeft: case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: @@ -799,22 +874,46 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kSubtract: case HloOpcode::kTanh: case HloOpcode::kRng: - case HloOpcode::kBroadcast: - case HloOpcode::kTranspose: + // De-emphasize scalar-shaped elementwise ops -- they're generally + // uninteresting. + if (ShapeUtil::IsEffectiveScalar(instr->shape())) { + return kWhite; + } return kYellow; case HloOpcode::kBitcast: case HloOpcode::kTuple: case HloOpcode::kTrace: case HloOpcode::kGetTupleElement: return kWhite; + case HloOpcode::kBroadcast: + // De-emphasize nodes which broadcast a scalar within a fusion node -- + // these are essentially free. + if (instr->IsFused() && + ShapeUtil::IsEffectiveScalar(instr->operand(0)->shape())) { + return kWhite; + } + return kGreen; case HloOpcode::kConcatenate: case HloOpcode::kCopy: case HloOpcode::kDynamicSlice: - case HloOpcode::kDynamicUpdateSlice: case HloOpcode::kPad: case HloOpcode::kReshape: case HloOpcode::kReverse: - case HloOpcode::kUpdate: + case HloOpcode::kSelect: + case HloOpcode::kTranspose: + // De-emphasize scalar-shaped data movement ops and all data movement ops + // inside fusion nodes, both of which are essentially free. + if (ShapeUtil::IsEffectiveScalar(instr->shape()) || instr->IsFused()) { + return kWhite; + } + return kGreen; + case HloOpcode::kDynamicUpdateSlice: + // Unlike the data-movement ops above, dynamic-update-slice is not ~free + // inside of fusion nodes, so we de-emphasize it only if it's + // scalar-shaped. + if (ShapeUtil::IsEffectiveScalar(instr->shape())) { + return kWhite; + } return kGreen; case HloOpcode::kConvolution: case HloOpcode::kDot: @@ -822,7 +921,7 @@ ColorScheme HloDotDumper::GetInstructionColor(const HloInstruction* instr) { case HloOpcode::kReducePrecision: return kRed; case HloOpcode::kParameter: - return kOrange; + return kParameterColor; case HloOpcode::kBatchNormTraining: case HloOpcode::kBatchNormInference: case HloOpcode::kBatchNormGrad: @@ -927,6 +1026,9 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { [](int64 stride) { return stride == 1; }) ? "" : StrCat("stride=", VectorString(instr->slice_strides())); + case HloOpcode::kSend: + case HloOpcode::kRecv: + return StrCat("channel_id=", instr->channel_id()); default: return ""; } @@ -936,7 +1038,9 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { if (!opcode_specific_info.empty()) { lines.push_back(opcode_specific_info); } - + if (instr->has_sharding()) { + lines.push_back(StrCat("sharding=", instr->sharding().ToString())); + } // Show the shape and layout of the instruction, unless it's an inlined fusion // node -- there the shape and layout is present in the output node. if (instr->opcode() != HloOpcode::kFusion || @@ -981,14 +1085,10 @@ string HloDotDumper::GetInstructionNodeExtraInfo(const HloInstruction* instr) { void HloDotDumper::AddInstructionIncomingEdges(const HloInstruction* instr) { auto add_edge = [&](const HloInstruction* from, const HloInstruction* to, int64 operand_num, bool control_edge = false) { - // Fusion nodes' subcomputations are displayed inline, so if 'from' is a - // fusion node and the node's subcomputation is shown, we draw our edge - // starting at the fusion node's root instead of at the fusion node itself. - if (from->opcode() == HloOpcode::kFusion && - ShouldShowFusionSubcomputation(from)) { - from = from->fused_expression_root(); - } - if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant) { + from = GetNodeForEdge(from); + + if (!filter_.Show(from) || from->opcode() == HloOpcode::kConstant || + ShouldMergeIntoUsers(from)) { return; } VLOG(2) << "Adding edge from " << from->name() << " to " << to->name() @@ -1054,6 +1154,15 @@ string HloDotDumper::GetInstructionTrivialComputationStr( return Join(lines, "
"); } +const HloInstruction* HloDotDumper::GetNodeForEdge( + const HloInstruction* instr) { + while (instr->opcode() == HloOpcode::kFusion && + ShouldShowFusionSubcomputation(instr)) { + instr = instr->fused_expression_root(); + } + return instr; +} + tensorflow::mutex& RendererMutex() { static tensorflow::mutex* mu = new tensorflow::mutex; return *mu; diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 021e5881c8af17de747b3189e7aae1d620a1035c..e6a4f68fb38001a65ea4d9d0b2b1ddaca4d85106 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -219,10 +219,12 @@ HloInstruction::CreateGetTupleElement(const Shape& shape, case HloOpcode::kCos: case HloOpcode::kExp: case HloOpcode::kFloor: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kSort: @@ -241,26 +243,28 @@ HloInstruction::CreateGetTupleElement(const Shape& shape, // Only certain opcodes are supported with CreateBinary: opcodes of binary // instructions with no auxiliary fields. switch (opcode) { - case (HloOpcode::kAdd): - case (HloOpcode::kDivide): - case (HloOpcode::kDot): - case (HloOpcode::kEq): - case (HloOpcode::kGe): - case (HloOpcode::kGt): - case (HloOpcode::kLe): - case (HloOpcode::kLt): - case (HloOpcode::kMaximum): - case (HloOpcode::kMinimum): - case (HloOpcode::kMultiply): - case (HloOpcode::kNe): - case (HloOpcode::kPower): - case (HloOpcode::kRemainder): - case (HloOpcode::kSubtract): - case (HloOpcode::kAnd): - case (HloOpcode::kOr): - case (HloOpcode::kShiftLeft): - case (HloOpcode::kShiftRightArithmetic): - case (HloOpcode::kShiftRightLogical): + case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kDivide: + case HloOpcode::kComplex: + case HloOpcode::kDot: + case HloOpcode::kEq: + case HloOpcode::kGe: + case HloOpcode::kGt: + case HloOpcode::kLe: + case HloOpcode::kLt: + case HloOpcode::kMaximum: + case HloOpcode::kMinimum: + case HloOpcode::kMultiply: + case HloOpcode::kNe: + case HloOpcode::kPower: + case HloOpcode::kRemainder: + case HloOpcode::kSubtract: + case HloOpcode::kAnd: + case HloOpcode::kOr: + case HloOpcode::kShiftLeft: + case HloOpcode::kShiftRightArithmetic: + case HloOpcode::kShiftRightLogical: break; default: LOG(FATAL) << "Invalid binary instruction opcode " @@ -716,10 +720,12 @@ void HloInstruction::MergeFusionInstructionIntoMultiOutput( // Fuse the root instruction and generate multiple outputs. FuseInstructionIntoMultiOutput(unfused_root); + TF_CHECK_OK(unfused_root->parent()->RemoveInstruction(unfused_root)); // The rest instructions are of normal fusing. for (int64 i = 1; i < unfused_instructions.size(); i++) { auto instruction = unfused_instructions[i]; FuseInstruction(instruction); + TF_CHECK_OK(instruction->parent()->RemoveInstruction(instruction)); } } @@ -962,6 +968,8 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( VLOG(3) << " " << new_operand->name(); } + std::unique_ptr clone; + // Explicitly call the factory for the instruction type. This is more robust // in the face of code changes than copying fields explicitly. This also // properly sets the user fields of the operands. @@ -974,19 +982,24 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kCopy: case HloOpcode::kCos: case HloOpcode::kExp: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kFloor: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kSign: case HloOpcode::kSin: case HloOpcode::kSort: case HloOpcode::kTanh: CHECK_EQ(new_operands.size(), 1); - return CreateUnary(shape, opcode_, new_operands[0]); + clone = CreateUnary(shape, opcode_, new_operands[0]); + break; // Binary ops. case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kMultiply: case HloOpcode::kSubtract: @@ -1007,125 +1020,155 @@ std::unique_ptr HloInstruction::CloneWithNewOperands( case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: CHECK_EQ(new_operands.size(), 2); - return CreateBinary(shape, opcode_, new_operands[0], new_operands[1]); + clone = CreateBinary(shape, opcode_, new_operands[0], new_operands[1]); + break; // Ternary ops. case HloOpcode::kClamp: case HloOpcode::kSelect: CHECK_EQ(new_operands.size(), 3); - return CreateTernary(shape, opcode_, new_operands[0], new_operands[1], - new_operands[2]); + clone = CreateTernary(shape, opcode_, new_operands[0], new_operands[1], + new_operands[2]); + break; // Other supported ops. case HloOpcode::kBroadcast: CHECK_EQ(new_operands.size(), 1); - return CreateBroadcast(shape, new_operands[0], dimensions_); + clone = CreateBroadcast(shape, new_operands[0], dimensions_); + break; case HloOpcode::kCall: - return CreateCall(shape, new_operands, to_apply()); + clone = CreateCall(shape, new_operands, to_apply()); + break; case HloOpcode::kCustomCall: - return CreateCustomCall(shape, new_operands, custom_call_target_); + clone = CreateCustomCall(shape, new_operands, custom_call_target_); + break; case HloOpcode::kConcatenate: - return CreateConcatenate(shape, new_operands, dimensions(0)); + clone = CreateConcatenate(shape, new_operands, dimensions(0)); + break; case HloOpcode::kConvert: CHECK_EQ(new_operands.size(), 1); - return CreateConvert(shape, new_operands[0]); + clone = CreateConvert(shape, new_operands[0]); + break; case HloOpcode::kReducePrecision: CHECK_EQ(new_operands.size(), 1); - return CreateReducePrecision(shape, new_operands[0], exponent_bits_, - mantissa_bits_); + clone = CreateReducePrecision(shape, new_operands[0], exponent_bits_, + mantissa_bits_); + break; case HloOpcode::kConvolution: CHECK_EQ(new_operands.size(), 2); - return CreateConvolve(shape, new_operands[0], new_operands[1], *window_, - *convolution_dimension_numbers_); + clone = CreateConvolve(shape, new_operands[0], new_operands[1], *window_, + *convolution_dimension_numbers_); + break; case HloOpcode::kCrossReplicaSum: CHECK_EQ(new_operands.size(), 1); - return CreateCrossReplicaSum(shape, new_operands[0]); + clone = CreateCrossReplicaSum(shape, new_operands[0]); + break; case HloOpcode::kGetTupleElement: CHECK_EQ(new_operands.size(), 1); - return CreateGetTupleElement(shape, new_operands[0], tuple_index()); + clone = CreateGetTupleElement(shape, new_operands[0], tuple_index()); + break; case HloOpcode::kMap: - return CreateMap(shape, new_operands, to_apply()); + clone = CreateMap(shape, new_operands, to_apply()); + break; case HloOpcode::kPad: CHECK_EQ(new_operands.size(), 2); - return CreatePad(shape, new_operands[0], new_operands[1], - *padding_config_); + clone = + CreatePad(shape, new_operands[0], new_operands[1], *padding_config_); + break; case HloOpcode::kReduce: CHECK_EQ(new_operands.size(), 2); - return CreateReduce(shape, new_operands[0], new_operands[1], dimensions_, - to_apply()); + clone = CreateReduce(shape, new_operands[0], new_operands[1], dimensions_, + to_apply()); + break; case HloOpcode::kReduceWindow: CHECK_EQ(new_operands.size(), 2); - return CreateReduceWindow(shape, new_operands[0], new_operands[1], - *window_, to_apply()); + clone = CreateReduceWindow(shape, new_operands[0], new_operands[1], + *window_, to_apply()); + break; case HloOpcode::kSelectAndScatter: CHECK_EQ(new_operands.size(), 3); - return CreateSelectAndScatter(shape, new_operands[0], select(), *window_, - new_operands[1], new_operands[2], - scatter()); + clone = + CreateSelectAndScatter(shape, new_operands[0], select(), *window_, + new_operands[1], new_operands[2], scatter()); + break; case HloOpcode::kReverse: CHECK_EQ(new_operands.size(), 1); - return CreateReverse(shape, new_operands[0], dimensions_); + clone = CreateReverse(shape, new_operands[0], dimensions_); + break; case HloOpcode::kRng: - return CreateRng(shape, distribution_, new_operands); + clone = CreateRng(shape, distribution_, new_operands); + break; case HloOpcode::kReshape: CHECK_EQ(new_operands.size(), 1); - return CreateReshape(shape, new_operands[0]); + clone = CreateReshape(shape, new_operands[0]); + break; case HloOpcode::kSlice: CHECK_EQ(new_operands.size(), 1); - return CreateSlice(shape, new_operands[0], slice_starts_, slice_limits_, - slice_strides_); + clone = CreateSlice(shape, new_operands[0], slice_starts_, slice_limits_, + slice_strides_); + break; case HloOpcode::kDynamicSlice: - return CreateDynamicSlice(shape, new_operands[0], new_operands[1], - dynamic_slice_sizes_); + clone = CreateDynamicSlice(shape, new_operands[0], new_operands[1], + dynamic_slice_sizes_); + break; case HloOpcode::kDynamicUpdateSlice: CHECK_EQ(new_operands.size(), 3); - return CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1], - new_operands[2]); + clone = CreateDynamicUpdateSlice(shape, new_operands[0], new_operands[1], + new_operands[2]); + break; case HloOpcode::kTranspose: CHECK_EQ(new_operands.size(), 1); - return CreateTranspose(shape, new_operands[0], dimensions_); - case HloOpcode::kTuple: { - auto new_tuple = CreateTuple(new_operands); - *new_tuple->mutable_shape() = shape; - return new_tuple; - } + clone = CreateTranspose(shape, new_operands[0], dimensions_); + break; + case HloOpcode::kTuple: + clone = CreateTuple(new_operands); + *clone->mutable_shape() = shape; + break; case HloOpcode::kWhile: CHECK_EQ(new_operands.size(), 1); - return CreateWhile(shape, while_condition(), while_body(), - new_operands[0]); + clone = + CreateWhile(shape, while_condition(), while_body(), new_operands[0]); + break; case HloOpcode::kConstant: - return CreateConstant(literal_->CloneToUnique()); + clone = CreateConstant(literal_->CloneToUnique()); + break; case HloOpcode::kFusion: - return CloneFusionWithNewOperands(shape, new_operands); + clone = CloneFusionWithNewOperands(shape, new_operands); + break; case HloOpcode::kParameter: - return CreateParameter(parameter_number_, shape, parameter_name_); + clone = CreateParameter(parameter_number_, shape, parameter_name_); + break; case HloOpcode::kBatchNormTraining: CHECK_EQ(new_operands.size(), 3); - return CreateBatchNormTraining(shape, new_operands[0], new_operands[1], - new_operands[2], epsilon(), - feature_index()); - + clone = + CreateBatchNormTraining(shape, new_operands[0], new_operands[1], + new_operands[2], epsilon(), feature_index()); + break; case HloOpcode::kBatchNormInference: CHECK_EQ(new_operands.size(), 5); - return CreateBatchNormInference( + clone = CreateBatchNormInference( shape, new_operands[0], new_operands[1], new_operands[2], new_operands[3], new_operands[4], epsilon(), feature_index()); + break; case HloOpcode::kInfeed: CHECK_EQ(new_operands.size(), 0); - return CreateInfeed(shape, infeed_config()); + clone = CreateInfeed(shape, infeed_config()); + break; case HloOpcode::kOutfeed: CHECK_EQ(new_operands.size(), 1); - return CreateOutfeed(outfeed_shape_, new_operands[0], outfeed_config()); + clone = CreateOutfeed(outfeed_shape_, new_operands[0], outfeed_config()); + break; case HloOpcode::kBatchNormGrad: CHECK_EQ(new_operands.size(), 5); - return CreateBatchNormGrad(shape, new_operands[0], new_operands[1], - new_operands[2], new_operands[3], - new_operands[4], epsilon(), feature_index()); + clone = CreateBatchNormGrad(shape, new_operands[0], new_operands[1], + new_operands[2], new_operands[3], + new_operands[4], epsilon(), feature_index()); + break; case HloOpcode::kRecv: case HloOpcode::kSend: - case HloOpcode::kUpdate: - case HloOpcode::kIndex: case HloOpcode::kTrace: LOG(FATAL) << "Not yet implemented, clone: " << HloOpcodeString(opcode_); } + clone->set_metadata(metadata_); + return clone; } HloInstruction::~HloInstruction() {} @@ -1168,7 +1211,9 @@ std::unique_ptr HloInstruction::Clone( } } clone->set_parent(parent_); - clone->set_metadata(metadata_); + if (has_sharding()) { + clone->set_sharding(sharding()); + } return clone; } @@ -1368,10 +1413,12 @@ bool HloInstruction::IdenticalSlowPath( // The result of these instructions only depend upon their opcode and // operands. case HloOpcode::kAbs: + case HloOpcode::kAtan2: case HloOpcode::kRoundNearestAfz: case HloOpcode::kAdd: case HloOpcode::kCeil: case HloOpcode::kClamp: + case HloOpcode::kComplex: case HloOpcode::kCopy: case HloOpcode::kCos: case HloOpcode::kCrossReplicaSum: @@ -1382,6 +1429,7 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kFloor: case HloOpcode::kGe: case HloOpcode::kGt: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLe: case HloOpcode::kLog: @@ -1395,6 +1443,7 @@ bool HloInstruction::IdenticalSlowPath( case HloOpcode::kNe: case HloOpcode::kNegate: case HloOpcode::kPower: + case HloOpcode::kReal: case HloOpcode::kRemainder: case HloOpcode::kSelect: case HloOpcode::kShiftLeft: @@ -1501,11 +1550,9 @@ bool HloInstruction::IdenticalSlowPath( return dimensions() == other.dimensions(); // These opcodes are not yet supported. - case HloOpcode::kIndex: case HloOpcode::kInfeed: case HloOpcode::kOutfeed: case HloOpcode::kSort: - case HloOpcode::kUpdate: case HloOpcode::kSend: case HloOpcode::kRecv: return false; @@ -1819,16 +1866,20 @@ std::vector HloInstruction::ExtraAttributesToString() const { } if (opcode() == HloOpcode::kWhile) { - extra.push_back(StrCat("condition=", while_condition()->name())); - extra.push_back(StrCat("body=", while_body()->name())); + extra.push_back(StrCat("condition=%", while_condition()->name())); + extra.push_back(StrCat("body=%", while_body()->name())); } else if (opcode() == HloOpcode::kSelectAndScatter) { - extra.push_back(StrCat("select=", select()->name())); - extra.push_back(StrCat("scatter=", scatter()->name())); + extra.push_back(StrCat("select=%", select()->name())); + extra.push_back(StrCat("scatter=%", scatter()->name())); + } else if (opcode() == HloOpcode::kCall || opcode() == HloOpcode::kMap || + opcode() == HloOpcode::kReduceWindow || + opcode() == HloOpcode::kReduce) { + extra.push_back(StrCat("to_apply=%", to_apply()->name())); } else if (!called_computations().empty()) { extra.push_back(StrCat( "calls=", Join(called_computations(), ", ", [](string* out, const HloComputation* computation) { - StrAppend(out, computation->name()); + StrAppend(out, "%", computation->name()); }))); } @@ -1839,6 +1890,9 @@ std::vector HloInstruction::ExtraAttributesToString() const { if (opcode() == HloOpcode::kGetTupleElement) { extra.push_back(StrCat("index=", tuple_index())); } + if (has_sharding()) { + extra.push_back(StrCat("sharding=", sharding().ToString())); + } if (!control_successors_.empty()) { extra.push_back(StrCat( "control-successors=", @@ -2075,7 +2129,9 @@ HloInstruction::HloInstruction(HloOpcode opcode, const Shape& shape) Status HloInstruction::Visit(DfsHloVisitor* visitor) { switch (opcode_) { case HloOpcode::kAbs: - return visitor->HandleAbs(this, operands_[0]); + return visitor->HandleAbs(this); + case HloOpcode::kAtan2: + return visitor->HandleAtan2(this); case HloOpcode::kRoundNearestAfz: return visitor->HandleRound(this); case HloOpcode::kBatchNormTraining: @@ -2085,11 +2141,11 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kBatchNormGrad: return visitor->HandleBatchNormGrad(this); case HloOpcode::kSign: - return visitor->HandleSign(this, operands_[0]); + return visitor->HandleSign(this); case HloOpcode::kConstant: - return visitor->HandleConstant(this, *literal_); + return visitor->HandleConstant(this); case HloOpcode::kGetTupleElement: - return visitor->HandleGetTupleElement(this, operands_[0]); + return visitor->HandleGetTupleElement(this); case HloOpcode::kParameter: return visitor->HandleParameter(this); case HloOpcode::kEq: @@ -2098,85 +2154,85 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kLe: case HloOpcode::kLt: case HloOpcode::kNe: - return visitor->HandleCompare(this, opcode_, operands_[0], operands_[1]); + return visitor->HandleCompare(this); + case HloOpcode::kComplex: + return visitor->HandleComplex(this); case HloOpcode::kAdd: - return visitor->HandleAdd(this, operands_[0], operands_[1]); + return visitor->HandleAdd(this); case HloOpcode::kDivide: - return visitor->HandleDivide(this, operands_[0], operands_[1]); + return visitor->HandleDivide(this); case HloOpcode::kSubtract: - return visitor->HandleSubtract(this, operands_[0], operands_[1]); + return visitor->HandleSubtract(this); case HloOpcode::kMaximum: return visitor->HandleMaximum(this); case HloOpcode::kMinimum: return visitor->HandleMinimum(this); case HloOpcode::kAnd: - return visitor->HandleAnd(this, operands_[0], operands_[1]); + return visitor->HandleAnd(this); case HloOpcode::kOr: - return visitor->HandleOr(this, operands_[0], operands_[1]); + return visitor->HandleOr(this); case HloOpcode::kShiftLeft: - return visitor->HandleShiftLeft(this, operands_[0], operands_[1]); + return visitor->HandleShiftLeft(this); case HloOpcode::kShiftRightArithmetic: - return visitor->HandleShiftRightArithmetic(this, operands_[0], - operands_[1]); + return visitor->HandleShiftRightArithmetic(this); case HloOpcode::kShiftRightLogical: - return visitor->HandleShiftRightLogical(this, operands_[0], operands_[1]); + return visitor->HandleShiftRightLogical(this); case HloOpcode::kConcatenate: - return visitor->HandleConcatenate(this, operands_); + return visitor->HandleConcatenate(this); case HloOpcode::kConvert: return visitor->HandleConvert(this); case HloOpcode::kCopy: return visitor->HandleCopy(this); case HloOpcode::kMultiply: - return visitor->HandleMultiply(this, operands_[0], operands_[1]); + return visitor->HandleMultiply(this); case HloOpcode::kDot: - return visitor->HandleDot(this, operands_[0], operands_[1]); + return visitor->HandleDot(this); case HloOpcode::kPower: - return visitor->HandlePower(this, operands_[0], operands_[1]); + return visitor->HandlePower(this); case HloOpcode::kRemainder: - return visitor->HandleRemainder(this, operands_[0], operands_[1]); + return visitor->HandleRemainder(this); case HloOpcode::kSelect: - return visitor->HandleSelect(this, operands_[0], operands_[1], - operands_[2]); + return visitor->HandleSelect(this); case HloOpcode::kConvolution: - return visitor->HandleConvolution(this, operands_[0], operands_[1], - window()); + return visitor->HandleConvolution(this); case HloOpcode::kCrossReplicaSum: return visitor->HandleCrossReplicaSum(this); case HloOpcode::kTuple: - return visitor->HandleTuple(this, operands_); + return visitor->HandleTuple(this); case HloOpcode::kMap: - return visitor->HandleMap(this, operands_, to_apply(), {}); + return visitor->HandleMap(this); case HloOpcode::kClamp: - return visitor->HandleClamp(this, operands_[0], operands_[1], - operands_[2]); + return visitor->HandleClamp(this); case HloOpcode::kReduce: - return visitor->HandleReduce(this, operands_[0], operands_[1], - dimensions_, to_apply()); + return visitor->HandleReduce(this); case HloOpcode::kReduceWindow: - return visitor->HandleReduceWindow(this, operands_[0], window(), - to_apply()); + return visitor->HandleReduceWindow(this); case HloOpcode::kSelectAndScatter: return visitor->HandleSelectAndScatter(this); case HloOpcode::kNegate: - return visitor->HandleNegate(this, operands_[0]); + return visitor->HandleNegate(this); case HloOpcode::kExp: - return visitor->HandleExp(this, operands_[0]); + return visitor->HandleExp(this); case HloOpcode::kFloor: - return visitor->HandleFloor(this, operands_[0]); + return visitor->HandleFloor(this); case HloOpcode::kCeil: - return visitor->HandleCeil(this, operands_[0]); + return visitor->HandleCeil(this); case HloOpcode::kLog: - return visitor->HandleLog(this, operands_[0]); + return visitor->HandleLog(this); case HloOpcode::kTanh: - return visitor->HandleTanh(this, operands_[0]); + return visitor->HandleTanh(this); case HloOpcode::kCos: - return visitor->HandleCos(this, operands_[0]); + return visitor->HandleCos(this); case HloOpcode::kSin: - return visitor->HandleSin(this, operands_[0]); + return visitor->HandleSin(this); + case HloOpcode::kReal: + return visitor->HandleReal(this); + case HloOpcode::kImag: + return visitor->HandleImag(this); case HloOpcode::kIsFinite: - return visitor->HandleIsFinite(this, operands_[0]); + return visitor->HandleIsFinite(this); case HloOpcode::kNot: - return visitor->HandleNot(this, operands_[0]); + return visitor->HandleNot(this); case HloOpcode::kBitcast: return visitor->HandleBitcast(this); case HloOpcode::kBroadcast: @@ -2188,24 +2244,23 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kTranspose: return visitor->HandleTranspose(this); case HloOpcode::kReverse: - return visitor->HandleReverse(this, operands_[0]); + return visitor->HandleReverse(this); case HloOpcode::kReducePrecision: return visitor->HandleReducePrecision(this); case HloOpcode::kSlice: - return visitor->HandleSlice(this, operands_[0]); + return visitor->HandleSlice(this); case HloOpcode::kDynamicSlice: - return visitor->HandleDynamicSlice(this, operands_[0], operands_[1]); + return visitor->HandleDynamicSlice(this); case HloOpcode::kDynamicUpdateSlice: - return visitor->HandleDynamicUpdateSlice(this, operands_[0], operands_[1], - operands_[2]); + return visitor->HandleDynamicUpdateSlice(this); case HloOpcode::kSort: - return visitor->HandleSort(this, operands_[0]); + return visitor->HandleSort(this); case HloOpcode::kInfeed: return visitor->HandleInfeed(this); case HloOpcode::kOutfeed: return visitor->HandleOutfeed(this); case HloOpcode::kRng: - return visitor->HandleRng(this, distribution_); + return visitor->HandleRng(this); case HloOpcode::kWhile: return visitor->HandleWhile(this); case HloOpcode::kFusion: @@ -2213,16 +2268,14 @@ Status HloInstruction::Visit(DfsHloVisitor* visitor) { case HloOpcode::kCall: return visitor->HandleCall(this); case HloOpcode::kCustomCall: - return visitor->HandleCustomCall(this, operands_, custom_call_target_); + return visitor->HandleCustomCall(this); case HloOpcode::kSend: return visitor->HandleSend(this); case HloOpcode::kRecv: return visitor->HandleRecv(this); // These opcodes are not handled here. - case HloOpcode::kIndex: case HloOpcode::kTrace: - case HloOpcode::kUpdate: break; } return Unimplemented("unhandled HloOpcode for DfsHloVisitor: %s", @@ -2265,7 +2318,7 @@ static Status PostOrderDFS(HloInstruction* root, DfsHloVisitor* visitor, // // We need to keep track of both the id and the instruction because // instructions can get deleted while they are on the stack, so we - // can't always use the (potentiall dead) instruction object to grab + // can't always use the (potentially dead) instruction object to grab // its id. DFSStack dfs_stack; dfs_stack.emplace_back(root->unique_id(), root); @@ -2465,6 +2518,7 @@ bool HloInstruction::IsElementwiseBinary() const { // Binary elementwise operations. If you update this, please update // IsElementwise() accordingly. case HloOpcode::kAdd: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kEq: case HloOpcode::kGe: @@ -2497,6 +2551,7 @@ bool HloInstruction::IsElementwise() const { // Unary elementwise operations. case HloOpcode::kAbs: + case HloOpcode::kAtan2: case HloOpcode::kRoundNearestAfz: case HloOpcode::kCeil: case HloOpcode::kConvert: @@ -2504,10 +2559,12 @@ bool HloInstruction::IsElementwise() const { case HloOpcode::kCos: case HloOpcode::kExp: case HloOpcode::kFloor: + case HloOpcode::kImag: case HloOpcode::kIsFinite: case HloOpcode::kLog: case HloOpcode::kNot: case HloOpcode::kNegate: + case HloOpcode::kReal: case HloOpcode::kReducePrecision: case HloOpcode::kSign: case HloOpcode::kSin: @@ -2517,6 +2574,7 @@ bool HloInstruction::IsElementwise() const { // Binary elementwise operations, the same as in IsElementwiseBinary(). // If you update this, please update IsElementwiseBinary() accordingly. case HloOpcode::kAdd: + case HloOpcode::kComplex: case HloOpcode::kDivide: case HloOpcode::kEq: case HloOpcode::kGe: @@ -2625,10 +2683,10 @@ class HloInstruction::FusionReusesParamElements { public: using UseKind = HloInstruction::UseKind; - // We could rather iterate backwards thru fused_instructions_ here, as it is - // in reverse postorder, and compute whether each fused instruction reuses - // the value of this parameter, which would save stack space but not allow - // us to finish early if we find a reuse. + // We could rather iterate backwards through fused_instructions_ here, as it + // is in reverse postorder, and compute whether each fused instruction reuses + // the value of this parameter, which would save stack space but not allow us + // to finish early if we find a reuse. static UseKind Compute(int64 i, const HloInstruction& hlo) { tensorflow::gtl::FlatMap memoization_cache; return ComputeInternal(i, hlo, &memoization_cache); diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index d2a15b0f962317cc79ab93cf377a77939a5eba41..e714d7bc71d86815b1b2df44cdd5c67281cdeb62 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/service/hlo_sharding.h" #include "tensorflow/compiler/xla/service/name_uniquer.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" @@ -713,6 +714,26 @@ class HloInstruction { fusion_kind_ = kind; } + // Returns the sharding applied to this operator. + // REQUIRES: has_sharding() is true. + const HloSharding& sharding() const { + CHECK(has_sharding()); + return *sharding_; + } + // Returns the sharding applied to this operator, or default_ if none exists. + const HloSharding& sharding_or_default(const HloSharding& default_) const { + return sharding_ ? *sharding_ : default_; + } + // Sets the sharding of this operator. Should only be called by HloModule or + // HloComputation methods. + void set_sharding(const HloSharding& sharding) { + sharding_ = MakeUnique(sharding); + } + // Remove any sharding from this operator. + void clear_sharding() { sharding_ = nullptr; } + // Return true if this operator has a sharding assigned. + bool has_sharding() const { return sharding_ != nullptr; } + // Merges the fused instructions from 'instruction_to_merge' into the // fused instruction set of 'this', updating operands as necessary. // @@ -984,14 +1005,6 @@ class HloInstruction { void RelayoutConstant(const Layout& new_layout, const ShapeIndex& shape_index = {}); - // Gets/sets the device assignment. - const OpDeviceAssignment& device_assignment() const { - return device_assignment_; - } - void set_device_assignment(const OpDeviceAssignment& device_assignment) { - device_assignment_ = device_assignment; - } - private: enum class UseKind { kNoUse, kReuse, kUsePermutingElements, kUse }; @@ -1124,6 +1137,9 @@ class HloInstruction { // The type of the fusion. Used by kFusion only. FusionKind fusion_kind_; + // The sharding, if one exists. + std::unique_ptr sharding_; + // For parameter instructions this field holds the parameter number. int64 parameter_number_ = 0; string parameter_name_; @@ -1184,9 +1200,6 @@ class HloInstruction { // outer-most dimension first). std::vector outer_dimension_partitions_; - // Device assignment for the instruction. - OpDeviceAssignment device_assignment_; - TF_DISALLOW_COPY_AND_ASSIGN(HloInstruction); }; diff --git a/tensorflow/compiler/xla/service/hlo_instruction_test.cc b/tensorflow/compiler/xla/service/hlo_instruction_test.cc index 45f9128eab766797030f8ab69700d8979e97f918..4ead64d997df1a6a85b028374949a4e5c9eab549 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction_test.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction_test.cc @@ -59,15 +59,15 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override { + Status HandleConstant(HloInstruction* constant) override { EXPECT_EQ(0, count_.count(constant)); count_[constant] = GetCountsForNode(constant); return Status::OK(); } - Status HandleAdd(HloInstruction* add, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleAdd(HloInstruction* add) override { + auto lhs = add->operand(0); + auto rhs = add->operand(1); EXPECT_EQ(0, count_.count(add)); EXPECT_GT(count_.count(lhs), 0); EXPECT_GT(count_.count(rhs), 0); @@ -75,32 +75,26 @@ class OpAndUserCollectingVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleNegate(HloInstruction* negate, - HloInstruction* operand) override { + Status HandleNegate(HloInstruction* negate) override { + auto operand = negate->operand(0); EXPECT_EQ(0, count_.count(negate)); EXPECT_GT(count_.count(operand), 0); count_[negate] = GetCountsForNode(negate); return Status::OK(); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* /*function*/, - tensorflow::gtl::ArraySlice /*static_operands*/) - override { + Status HandleMap(HloInstruction* map) override { EXPECT_EQ(0, count_.count(map)); - for (HloInstruction* arg : operands) { + for (HloInstruction* arg : map->operands()) { EXPECT_GT(count_.count(arg), 0); } count_[map] = GetCountsForNode(map); return Status::OK(); } - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override { + Status HandleReduce(HloInstruction* reduce) override { + auto arg = reduce->operand(0); + auto init_value = reduce->operand(1); EXPECT_EQ(0, count_.count(reduce)); EXPECT_GT(count_.count(arg), 0); EXPECT_GT(count_.count(init_value), 0); @@ -706,6 +700,9 @@ TEST_F(HloInstructionTest, PreserveMetadataInFusionAndClone) { metadata, fusion->fused_expression_root()->metadata())); EXPECT_TRUE(protobuf_util::ProtobufEquals( metadata, fusion->fused_expression_root()->operand(0)->metadata())); + + auto cloned = fusion->CloneWithNewOperands(fusion->shape(), {}); + EXPECT_TRUE(protobuf_util::ProtobufEquals(metadata, fusion->metadata())); } TEST_F(HloInstructionTest, PreserveOutfeedShapeThroughClone) { @@ -1200,13 +1197,13 @@ TEST_F(HloInstructionTest, Stringification) { EXPECT_EQ(fusion->ToString(false, false), "%fusion = f32[5,20]{1,0} fusion:kTransposeDot(f32[5,10]{1,0} %x, " - "f32[20,10]{1,0} %y), calls=fused_computation"); + "f32[20,10]{1,0} %y), calls=%fused_computation"); HloInstruction* loop = builder.AddInstruction( HloInstruction::CreateWhile(sout, computation, computation, x)); EXPECT_EQ(loop->ToString(false, false), "%while = f32[5,20]{1,0} while(f32[5,10]{1,0} %x), " - "condition=TransposeDot, body=TransposeDot"); + "condition=%TransposeDot, body=%TransposeDot"); } } // namespace diff --git a/tensorflow/compiler/xla/service/hlo_matchers.h b/tensorflow/compiler/xla/service/hlo_matchers.h index d1ae5f776d281aa4cad157c9e2bc1f2c1133b37f..bc5ed029a45b4f92a240138dc1e933610efe1789 100644 --- a/tensorflow/compiler/xla/service/hlo_matchers.h +++ b/tensorflow/compiler/xla/service/hlo_matchers.h @@ -74,7 +74,6 @@ HLO_MATCHER(Fusion); HLO_MATCHER(Ge); HLO_MATCHER(GetTupleElement); HLO_MATCHER(Gt); -HLO_MATCHER(Index); HLO_MATCHER(Infeed); HLO_MATCHER(IsFinite); HLO_MATCHER(Le); @@ -115,7 +114,6 @@ HLO_MATCHER(Tanh); HLO_MATCHER(Trace); HLO_MATCHER(Transpose); HLO_MATCHER(Tuple); -HLO_MATCHER(Update); HLO_MATCHER(While); #undef HLO_MATCHER } // namespace opcode_matchers diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 5bc7a3643936b3cb3ef066b4f741c934f5e850d3..1758f2760c46a5f0f5876ac6ba8dd013e71455b6 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -45,10 +45,37 @@ HloModule::HloModule(const string& name, const HloModuleConfig& config) : name_(name), config_(config) {} HloComputation* HloModule::AddComputationInternal( - std::unique_ptr computation) { - computation->UniquifyName(&computation_name_uniquer_); + std::unique_ptr computation, bool is_entry, + bool uniquify_names) { + if (is_entry) { + CHECK_EQ(nullptr, entry_computation_); + entry_computation_ = computation.get(); + + // If the module configuration has no entry layout computation set, create a + // default one based on the program shape. + if (!config_.has_entry_computation_layout()) { + config_.SetDefaultComputationLayout( + entry_computation_->ComputeProgramShape()); + } + } + + if (uniquify_names) { + computation->UniquifyName(&computation_name_uniquer_); + for (auto* instruction : computation->instructions()) { + instruction->UniquifyName(&instruction_name_uniquer_); + } + } else { + // Don't uniquify the names of the computation or instruction, but we must + // run the names through the uniquifiers to prevent future name collisions + // for computations and instructions created later. + computation_name_uniquer_.GetUniqueName(computation->name()); + for (auto* instruction : computation->instructions()) { + instruction_name_uniquer_.GetUniqueName(instruction->name()); + } + } + + // Pick unique IDs for each instruction. for (auto* instruction : computation->instructions()) { - instruction->UniquifyName(&instruction_name_uniquer_); instruction->SetUniqueId(NewUniqueInstructionId()); } computation->set_parent(this); @@ -58,16 +85,8 @@ HloComputation* HloModule::AddComputationInternal( HloComputation* HloModule::AddEntryComputation( std::unique_ptr computation) { - CHECK_EQ(nullptr, entry_computation_); - entry_computation_ = computation.get(); - - // If the module configuration has no entry layout computation set, create a - // default one based on the program shape. - if (!config_.has_entry_computation_layout()) { - config_.SetDefaultComputationLayout( - entry_computation_->ComputeProgramShape()); - } - return AddComputationInternal(std::move(computation)); + return AddComputationInternal(std::move(computation), /*is_entry=*/true, + /*uniquify_names=*/true); } Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) { @@ -83,7 +102,8 @@ Status HloModule::RemoveEmbeddedComputation(HloComputation* to_remove) { HloComputation* HloModule::AddEmbeddedComputation( std::unique_ptr computation) { - return AddComputationInternal(std::move(computation)); + return AddComputationInternal(std::move(computation), /*is_entry=*/false, + /*uniquify_names=*/true); } void HloModule::ReplaceComputations( @@ -153,11 +173,17 @@ void HloModule::ReplaceComputations( string HloModule::ToString() const { std::ostringstream s; s << "HloModule " << name() << ":\n\n"; - s << "ENTRY " << entry_computation()->ToString() << "\n\n"; - for (const std::unique_ptr& computation : computations_) { - if (computation.get() != entry_computation()) { - s << computation->ToString() << "\n\n"; + for (const HloComputation* computation : MakeComputationPostOrder()) { + // Fusion computations are emitted with their fusion instruction and + // therefore don't need to be emitted as a separate comptutation in the + // module. + if (computation->IsFusionComputation()) { + continue; } + if (computation == entry_computation()) { + s << "ENTRY "; + } + s << computation->ToString() << "\n\n"; } return s.str(); } @@ -178,13 +204,93 @@ HloModuleProto HloModule::ToProto() const { return proto; } +namespace { + +// Construct a ProgramShape matching the shape of the parameters and root of the +// given module's entry computation. +StatusOr ProgramShapeFromProto(const HloModuleProto& module) { + const HloComputationProto* entry_computation = nullptr; + for (const HloComputationProto& computation : module.computations()) { + if (computation.name() == module.entry_computation_name()) { + entry_computation = &computation; + break; + } + } + TF_RET_CHECK(entry_computation != nullptr) + << "No computation with entry computation name" + << module.entry_computation_name(); + + tensorflow::gtl::FlatMap> parameters; + const HloInstructionProto* root = nullptr; + for (const HloInstructionProto& instruction : + entry_computation->instructions()) { + if (instruction.name() == entry_computation->root_name()) { + TF_RET_CHECK(root == nullptr) << "Entry computation has more than " + "one instruction with (root) name " + << instruction.name(); + root = &instruction; + } + if (instruction.opcode() == HloOpcodeString(HloOpcode::kParameter)) { + TF_RET_CHECK(!ContainsKey(parameters, instruction.parameter_number())) + << "Entry computation has more than one parameter instruction " + "with parameter number " + << instruction.parameter_number(); + parameters[instruction.parameter_number()] = { + instruction.parameter_name(), &instruction.shape()}; + } + } + TF_RET_CHECK(root != nullptr) + << "Entry computation is missing root instruction named " + << entry_computation->root_name(); + + ProgramShape program_shape; + *program_shape.mutable_result() = root->shape(); + for (int64 i = 0; i < parameters.size(); ++i) { + TF_RET_CHECK(ContainsKey(parameters, i)) + << "Entry computation missing parameter number " << i; + const string& name = parameters.at(i).first; + const Shape& shape = *parameters.at(i).second; + *program_shape.add_parameters() = shape; + program_shape.add_parameter_names(name); + } + + return std::move(program_shape); +} + +} // namespace + /* static */ StatusOr> HloModule::CreateFromProto( - const HloModuleProto& proto, - const VersionedComputationHandle& entry_computation_handle, - const HloModuleConfig& config) { - auto module = - MakeUnique(proto.name(), entry_computation_handle, config); + const HloModuleProto& proto, const HloModuleConfig& module_config, + const VersionedComputationHandle& entry_computation_handle) { + // The ProgramShape in the passed in module config must match the shapes of + // the entry parameters and root. + TF_ASSIGN_OR_RETURN(ProgramShape expected_program_shape, + ProgramShapeFromProto(proto)); + TF_RET_CHECK(expected_program_shape.parameters_size() == + module_config.entry_computation_layout().parameter_count()); + for (int i = 0; i < expected_program_shape.parameters_size(); ++i) { + const Shape& parameter_shape = + module_config.entry_computation_layout().parameter_layout(i).shape(); + TF_RET_CHECK( + ShapeUtil::Equal(expected_program_shape.parameters(i), parameter_shape)) + << "HloModuleConfig has different shape for parameter " << i + << " than the HLO module. Expected: " + << ShapeUtil::HumanStringWithLayout( + expected_program_shape.parameters(i)) + << ", actual: " << ShapeUtil::HumanStringWithLayout(parameter_shape); + } + const Shape& result_shape = + module_config.entry_computation_layout().result_layout().shape(); + TF_RET_CHECK(ShapeUtil::Equal(expected_program_shape.result(), result_shape)) + << "HloModuleConfig has different result shape than the HLO module. " + "Expected: " + << ShapeUtil::HumanStringWithLayout(expected_program_shape.result()) + << ", actual: " << ShapeUtil::HumanStringWithLayout(result_shape); + + auto module = MakeUnique(proto.name(), entry_computation_handle, + module_config); + tensorflow::gtl::FlatMap computation_map; for (const HloComputationProto& computation_proto : proto.computations()) { TF_ASSIGN_OR_RETURN(std::unique_ptr computation, @@ -193,19 +299,60 @@ StatusOr> HloModule::CreateFromProto( CHECK_NE(computation.get(), nullptr); TF_RET_CHECK(!ContainsKey(computation_map, computation->name())); string computation_name = computation->name(); - if (proto.entry_computation_name() == computation_name) { - computation_map[computation_name] = - module->AddEntryComputation(std::move(computation)); - } else { - computation_map[computation_name] = - module->AddEmbeddedComputation(std::move(computation)); - } + // Don't uniquify names because we want names to be stable across + // serialization and deserialization. + computation_map[computation_name] = module->AddComputationInternal( + std::move(computation), + /*is_entry=*/proto.entry_computation_name() == computation_name, + /*uniquify_names=*/false); } TF_RET_CHECK(module->entry_computation_ != nullptr); + // Because we didn't uniquify the names, double-check that the instruction and + // computation names are unique from the proto. + tensorflow::gtl::FlatSet computation_names; + tensorflow::gtl::FlatSet instruction_names; + for (HloComputation* computation : module->computations()) { + if (computation->IsFusionComputation()) { + continue; + } + + TF_RET_CHECK(!ContainsKey(computation_names, computation->name())) + << "Computation name is not unique: " << computation->name(); + computation_names.insert(computation->name()); + for (HloInstruction* instruction : computation->instructions()) { + TF_RET_CHECK(!ContainsKey(instruction_names, instruction->name())) + << "Instruction name is not unique: " << instruction->name(); + instruction_names.insert(instruction->name()); + } + } + return std::move(module); } +/* static */ +StatusOr HloModule::CreateModuleConfigFromProto( + const HloModuleProto& module) { + TF_ASSIGN_OR_RETURN(ProgramShape program_shape, + ProgramShapeFromProto(module)); + + HloModuleConfig module_config(program_shape); + + // The module config is constructed with default layouts regardless of what is + // passed in via the ProgramShape. Set the layouts to the appropriate values. + ComputationLayout* entry_layout = + module_config.mutable_entry_computation_layout(); + for (int64 i = 0; i < entry_layout->parameter_count(); ++i) { + TF_RETURN_IF_ERROR( + entry_layout->mutable_parameter_layout(i)->CopyLayoutFromShape( + program_shape.parameters(i))); + } + TF_RETURN_IF_ERROR(entry_layout->mutable_result_layout()->CopyLayoutFromShape( + program_shape.result())); + + return module_config; +} + namespace { // Returns whether `hlo` is used outside the given subcomputation. // `instructions_in_subcomputation` is the instruction set of the given diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h index 96c17d62970d48ccf590c44115c61c89fd379efe..ad11d56006a79b509309daba55e94342911f76a1 100644 --- a/tensorflow/compiler/xla/service/hlo_module.h +++ b/tensorflow/compiler/xla/service/hlo_module.h @@ -144,9 +144,14 @@ class HloModule { // Convert an HloModule to or from a proto. HloModuleProto ToProto() const; static StatusOr> CreateFromProto( - const HloModuleProto& proto, - const VersionedComputationHandle& entry_computation_handle, - const HloModuleConfig& config); + const HloModuleProto& proto, const HloModuleConfig& module_config, + const VersionedComputationHandle& entry_computation_handle = + VersionedComputationHandle()); + + // Creates and returns an HloModuleConfig with an appropriate program shape + // for the HLO module in the given proto. + static StatusOr CreateModuleConfigFromProto( + const HloModuleProto& module); // Outlines the given expression from the given computation. // instructions_to_outline contains the instructions that form the expression. @@ -182,7 +187,8 @@ class HloModule { private: HloComputation* AddComputationInternal( - std::unique_ptr computation); + std::unique_ptr computation, bool is_entry, + bool uniquify_names); const string name_; HloModuleConfig config_; diff --git a/tensorflow/compiler/xla/service/hlo_opcode.cc b/tensorflow/compiler/xla/service/hlo_opcode.cc index db3abeab22044de372c6fb6237d7a4b859884ec9..157d19f5a9996ff90c4a5c3655f82ff5b8e62cfc 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.cc +++ b/tensorflow/compiler/xla/service/hlo_opcode.cc @@ -33,6 +33,10 @@ string HloOpcodeString(HloOpcode opcode) { return "abs"; case HloOpcode::kAdd: return "add"; + case HloOpcode::kAnd: + return "and"; + case HloOpcode::kAtan2: + return "atan2"; case HloOpcode::kBatchNormTraining: return "batch-norm-training"; case HloOpcode::kBatchNormInference: @@ -47,6 +51,8 @@ string HloOpcodeString(HloOpcode opcode) { return "call"; case HloOpcode::kClamp: return "clamp"; + case HloOpcode::kComplex: + return "complex"; case HloOpcode::kConcatenate: return "concatenate"; case HloOpcode::kConstant: @@ -87,8 +93,8 @@ string HloOpcodeString(HloOpcode opcode) { return "get-tuple-element"; case HloOpcode::kGt: return "greater-than"; - case HloOpcode::kIndex: - return "index"; + case HloOpcode::kImag: + return "imag"; case HloOpcode::kInfeed: return "infeed"; case HloOpcode::kIsFinite: @@ -97,12 +103,6 @@ string HloOpcodeString(HloOpcode opcode) { return "less-than-or-equal-to"; case HloOpcode::kLog: return "log"; - case HloOpcode::kAnd: - return "and"; - case HloOpcode::kOr: - return "or"; - case HloOpcode::kNot: - return "not"; case HloOpcode::kLt: return "less-than"; case HloOpcode::kMap: @@ -117,6 +117,10 @@ string HloOpcodeString(HloOpcode opcode) { return "not-equal-to"; case HloOpcode::kNegate: return "negate"; + case HloOpcode::kNot: + return "not"; + case HloOpcode::kOr: + return "or"; case HloOpcode::kOutfeed: return "outfeed"; case HloOpcode::kPad: @@ -125,6 +129,8 @@ string HloOpcodeString(HloOpcode opcode) { return "parameter"; case HloOpcode::kPower: return "power"; + case HloOpcode::kReal: + return "real"; case HloOpcode::kRecv: return "recv"; case HloOpcode::kReduce: @@ -173,8 +179,6 @@ string HloOpcodeString(HloOpcode opcode) { return "transpose"; case HloOpcode::kTuple: return "tuple"; - case HloOpcode::kUpdate: - return "update"; case HloOpcode::kWhile: return "while"; } @@ -184,6 +188,7 @@ StatusOr StringToHloOpcode(const string& opcode_name) { static auto* opcode_map = new tensorflow::gtl::FlatMap( {{"abs", HloOpcode::kAbs}, {"add", HloOpcode::kAdd}, + {"and", HloOpcode::kAnd}, {"batch-norm-training", HloOpcode::kBatchNormTraining}, {"batch-norm-inference", HloOpcode::kBatchNormInference}, {"batch-norm-grad", HloOpcode::kBatchNormGrad}, @@ -211,21 +216,19 @@ StatusOr StringToHloOpcode(const string& opcode_name) { {"greater-than-or-equal-to", HloOpcode::kGe}, {"get-tuple-element", HloOpcode::kGetTupleElement}, {"greater-than", HloOpcode::kGt}, - {"index", HloOpcode::kIndex}, {"infeed", HloOpcode::kInfeed}, {"is-finite", HloOpcode::kIsFinite}, {"less-than-or-equal-to", HloOpcode::kLe}, {"log", HloOpcode::kLog}, - {"and", HloOpcode::kAnd}, - {"or", HloOpcode::kOr}, - {"not", HloOpcode::kNot}, {"less-than", HloOpcode::kLt}, {"map", HloOpcode::kMap}, {"maximum", HloOpcode::kMaximum}, {"minimum", HloOpcode::kMinimum}, {"multiply", HloOpcode::kMultiply}, + {"not", HloOpcode::kNot}, {"not-equal-to", HloOpcode::kNe}, {"negate", HloOpcode::kNegate}, + {"or", HloOpcode::kOr}, {"outfeed", HloOpcode::kOutfeed}, {"pad", HloOpcode::kPad}, {"parameter", HloOpcode::kParameter}, @@ -254,7 +257,6 @@ StatusOr StringToHloOpcode(const string& opcode_name) { {"trace", HloOpcode::kTrace}, {"transpose", HloOpcode::kTranspose}, {"tuple", HloOpcode::kTuple}, - {"update", HloOpcode::kUpdate}, {"while", HloOpcode::kWhile}}); auto it = opcode_map->find(opcode_name); if (it == opcode_map->end()) { diff --git a/tensorflow/compiler/xla/service/hlo_opcode.h b/tensorflow/compiler/xla/service/hlo_opcode.h index 4593df671e34b1ec1f6e388439df37adf63b621f..07c2d26f00f2338d306b57933e5f0fb77b38b892 100644 --- a/tensorflow/compiler/xla/service/hlo_opcode.h +++ b/tensorflow/compiler/xla/service/hlo_opcode.h @@ -31,6 +31,7 @@ namespace xla { enum class HloOpcode { kAbs, kAdd, + kAtan2, kBatchNormGrad, kBatchNormInference, kBatchNormTraining, @@ -39,6 +40,7 @@ enum class HloOpcode { kCall, kCeil, kClamp, + kComplex, kConcatenate, kConstant, kConvert, @@ -58,7 +60,7 @@ enum class HloOpcode { kGe, kGetTupleElement, kGt, - kIndex, + kImag, kInfeed, kIsFinite, kLe, @@ -77,6 +79,7 @@ enum class HloOpcode { kPad, kParameter, kPower, + kReal, kRecv, kReduce, kReducePrecision, @@ -101,7 +104,6 @@ enum class HloOpcode { kTrace, kTranspose, kTuple, - kUpdate, kWhile, }; diff --git a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc index ed7b6c71bc6619b0cb93f226eb10de1023749109..53bd46a641afcba1b9551895955742e74a9f374b 100644 --- a/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc +++ b/tensorflow/compiler/xla/service/hlo_pass_pipeline.cc @@ -59,6 +59,7 @@ StatusOr HloPassPipeline::Run(HloModule* module) { for (auto& invariant_checker : invariant_checkers_) { VLOG(1) << " Invariant checker " << invariant_checker->name(); StatusOr changed_status = invariant_checker->Run(module); + VLOG(1) << " Invariant checker done " << invariant_checker->name(); if (!changed_status.ok()) { VLOG(2) << "Module failed invariant check:"; XLA_VLOG_LINES(2, module->ToString()); diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc new file mode 100644 index 0000000000000000000000000000000000000000..c3f74e253f7a7882ec1c72e0ce634017dd2f0957 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_runner.cc @@ -0,0 +1,178 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_runner.h" + +#include +#include +#include + +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/compiler/xla/layout_util.h" +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/backend.h" +#include "tensorflow/compiler/xla/service/executable.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/transfer_manager.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/core/common_runtime/eigen_thread_pool.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" + +namespace se = ::perftools::gputools; + +namespace xla { + +/*static*/ StatusOr> +HloRunner::ReadModuleFromHloProtoFile(const char* filename, + const DebugOptions& debug_options) { + HloProto proto; + TF_RETURN_IF_ERROR(tensorflow::ReadBinaryProto(tensorflow::Env::Default(), + filename, &proto)); + TF_ASSIGN_OR_RETURN( + HloModuleConfig config, + HloModule::CreateModuleConfigFromProto(proto.hlo_module())); + config.set_debug_options(debug_options); + TF_ASSIGN_OR_RETURN(auto module, + HloModule::CreateFromProto(proto.hlo_module(), config)); + return std::move(module); +} + +// Define this in .cc file to avoid having to include eigen or forward declare +// these types in the header. +struct HloRunner::EigenThreadPoolWrapper { + std::unique_ptr pool; + std::unique_ptr device; +}; + +HloRunner::HloRunner() {} + +HloRunner::HloRunner(se::Platform* platform) { + BackendOptions backend_options; + backend_options.set_platform(platform); + backend_ = Backend::CreateBackend(backend_options).ConsumeValueOrDie(); + VLOG(1) << "Created HloRunner for platform: " << platform->Name(); +} + +HloRunner::~HloRunner() { + // Deallocate all the memory allocated during the tests. + for (auto& allocation : allocations_) { + backend().default_stream_executor()->Deallocate(&allocation); + } +} + +StatusOr HloRunner::Execute( + std::unique_ptr module, + tensorflow::gtl::ArraySlice arguments, + Shape* result_shape) { + TF_ASSIGN_OR_RETURN( + std::unique_ptr executable, + backend().compiler()->Compile(std::move(module), + backend().default_stream_executor())); + + se::Stream stream(backend().default_stream_executor()); + stream.Init(); + + ExecutableRunOptions run_options; + run_options.set_stream(&stream); + run_options.set_allocator(backend().memory_allocator()); + run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool()); + run_options.set_intra_op_thread_pool( + backend().eigen_intra_op_thread_pool_device()); + + HloExecutionProfile hlo_execution_profile; + ServiceExecutableRunOptions service_run_options( + run_options, backend().StreamBorrower(), + backend().inter_op_thread_pool()); + TF_ASSIGN_OR_RETURN( + se::DeviceMemoryBase result, + executable->ExecuteOnStream(&service_run_options, arguments, + &hlo_execution_profile)); + TF_RET_CHECK(stream.BlockHostUntilDone()); + + allocations_.push_back(result); + + *result_shape = executable->result_shape(); + + if (ShapeUtil::IsTuple(*result_shape)) { + // We must record element buffers of tuples as well to avoid leaks. + DCHECK(!ShapeUtil::IsNestedTuple(*result_shape)); + TF_ASSIGN_OR_RETURN( + std::vector element_buffers, + backend().transfer_manager()->ShallowCopyTupleFromDevice( + backend().default_stream_executor(), result, *result_shape)); + + // A tuple may contain the same buffer in more than one element. Keep track + // of the buffers already added to avoid duplicates in allocations_. + std::set added_opaques; + for (auto element_buffer : element_buffers) { + if (added_opaques.count(element_buffer.opaque()) == 0) { + CHECK(element_buffer.opaque() != nullptr); + added_opaques.insert(element_buffer.opaque()); + allocations_.push_back(element_buffer); + } + } + } + + return result; +} + +StatusOr HloRunner::TransferToDevice( + const Literal& literal) { + // Allocate memory on the device using the stream executor. + int64 allocation_size = + backend().transfer_manager()->GetByteSizeRequirement(literal.shape()); + se::DeviceMemoryBase allocation = + backend().default_stream_executor()->AllocateArray( + allocation_size); + allocations_.push_back(allocation); + + TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralToDevice( + backend().default_stream_executor(), literal, &allocation)); + + return allocation; +} + +StatusOr> HloRunner::TransferFromDevice( + const Shape& shape, se::DeviceMemoryBase device_base) { + auto literal = MakeUnique(); + TF_RETURN_IF_ERROR(backend().transfer_manager()->TransferLiteralFromDevice( + backend().default_stream_executor(), device_base, shape, shape, + literal.get())); + return std::move(literal); +} + +StatusOr> HloRunner::ExecuteAndTransfer( + std::unique_ptr module, + tensorflow::gtl::ArraySlice arguments) { + Shape result_shape; + TF_ASSIGN_OR_RETURN(se::DeviceMemoryBase device_base, + Execute(std::move(module), arguments, &result_shape)); + return TransferFromDevice(result_shape, device_base); +} + +Backend& HloRunner::backend() { + if (!backend_) { + backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie(); + VLOG(1) << "executing on platform " << backend().platform()->Name(); + } + return *backend_; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_runner.h b/tensorflow/compiler/xla/service/hlo_runner.h new file mode 100644 index 0000000000000000000000000000000000000000..a4d7b653dbfbfdb169c07bca3e461147fd9d077a --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_runner.h @@ -0,0 +1,114 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ + +#include +#include +#include + +#include "tensorflow/compiler/xla/service/backend.h" +#include "tensorflow/compiler/xla/service/compiler.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/platform/stream_executor_no_cuda.h" + +namespace xla { + +// A base class for running an HloModule. This executes the given HloModule on a +// certain backend directly without using the client interface. HloModule can be +// explicitly built, or loaded from a serialization file (e.g., hlo proto file). +class HloRunner { + public: + HloRunner(); + + HloRunner(::perftools::gputools::Platform* platform); + + ~HloRunner(); + + // Reads the binary proto file in xla.HloProto format, creates and returns the + // HloModule. + static StatusOr> ReadModuleFromHloProtoFile( + const char* filename, const DebugOptions& debug_options); + + // Executes the given module with given literals as input and returns the + // result as a Literal. The LiteralPtr type accepts Literal* or + // std::unique_ptr. + template + StatusOr> Execute( + std::unique_ptr module, + const tensorflow::gtl::ArraySlice literals); + + // Executes the given module and returns a global data handle. + StatusOr Execute( + std::unique_ptr module, + tensorflow::gtl::ArraySlice + arguments, + Shape* result_shape); + + // Transfers the given literal to the device and returns the data handle. + StatusOr TransferToDevice( + const Literal& literal); + + // Transfers the array referred to by the given handle from the device and + // returns as a Literal. + StatusOr> TransferFromDevice( + const Shape& shape, perftools::gputools::DeviceMemoryBase device_base); + + // Executes the given module and return the result as a Literal. + StatusOr> ExecuteAndTransfer( + std::unique_ptr module, + tensorflow::gtl::ArraySlice + arguments); + + // If backend is not created in the constructor, creates and returns the + // default backend. If creation fails, crashes the program. + // + // This creates the backend lazily so it's possible to instantiate an + // HloRunner in a program without any backends linked in. + Backend& backend(); + + private: + struct EigenThreadPoolWrapper; + + std::vector allocations_; + + std::unique_ptr thread_pool_wrapper_; + + std::unique_ptr backend_; +}; + +template +StatusOr> HloRunner::Execute( + std::unique_ptr module, + const tensorflow::gtl::ArraySlice literals) { + std::vector arguments; + for (const auto& literal : literals) { + TF_ASSIGN_OR_RETURN(perftools::gputools::DeviceMemoryBase argument, + TransferToDevice(*literal)); + arguments.push_back(argument); + } + return ExecuteAndTransfer(std::move(module), arguments); +} + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_RUNNER_H_ diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc new file mode 100644 index 0000000000000000000000000000000000000000..0d019d22f5d4cd401c0fc5572f99636dec4f7383 --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -0,0 +1,232 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_sharding.h" + +#include "tensorflow/core/lib/core/errors.h" + +namespace xla { + +using ::tensorflow::strings::StrCat; + +HloSharding HloSharding::AssignDevice(int64 device_id) { + return HloSharding(device_id); +} + +HloSharding HloSharding::Tile1D(const Shape& input_shape, int64 num_tiles) { + CHECK_EQ(1, ShapeUtil::Rank(input_shape)); + CHECK_GT(num_tiles, 1); + std::vector dimensions(1, num_tiles); + Shape tile_shape = input_shape; + auto& tile_dimension = (*tile_shape.mutable_dimensions())[0]; + tile_dimension = CeilOfRatio(static_cast(tile_dimension), num_tiles); + Array assignment(dimensions); + std::iota(assignment.begin(), assignment.end(), 0); + return HloSharding(tile_shape, assignment); +} + +string HloSharding::ToString() const { + string result = StrCat("{", (replicated_ ? " replicated" : ""), + (maximal_ ? " maximal" : "")); + + if (replicated_) { + return "{replicated}"; + } else if (maximal_) { + return StrCat( + "{maximal device=", static_cast(*tile_assignment_.begin()), "}"); + } else { + return StrCat("{", ShapeUtil::HumanString(tile_shape_), " ", + "devices=", VectorString(tile_assignment_), "}"); + } +} + +bool HloSharding::UsesDevice(int64 device) const { + const auto& devices = tile_assignment_; + return replicated_ || + std::find(devices.begin(), devices.end(), device) != devices.end(); +} + +std::vector HloSharding::TileIndexForDevice(int64 device) const { + CHECK(!ShapeUtil::IsTuple(tile_shape_)); + CHECK(!maximal_); + std::vector ret_index; + tile_assignment_.Each([&](tensorflow::gtl::ArraySlice index, int64 d) { + if (d == device) { + ret_index = {index.begin(), index.end()}; + } + }); + CHECK(!ret_index.empty()); + return ret_index; +} + +int64 HloSharding::DeviceForTileIndex( + tensorflow::gtl::ArraySlice index) const { + CHECK(!replicated_); + if (maximal_) { + return *tile_assignment_.begin(); + } + CHECK_EQ(ShapeUtil::Rank(tile_shape_), tile_assignment_.dimensions().size()); + return tile_assignment_(index); +} + +std::vector HloSharding::TileOffsetForDevice(int64 device) const { + CHECK(!ShapeUtil::IsTuple(tile_shape_)); + + std::vector index = TileIndexForDevice(device); + if (maximal_) { + // Index will always be all zeroes if we're maximal, and tile_shape_ is not + // valid. + return index; + } + for (int64 i = 0; i < index.size(); ++i) { + index[i] *= tile_shape_.dimensions(i); + } + return index; +} + +std::vector HloSharding::TileLimitForDevice(int64 device) const { + CHECK(!ShapeUtil::IsTuple(tile_shape_)); + CHECK(!maximal_); // Maximal shardings do not have a valid tile shape. + + std::vector index = TileIndexForDevice(device); + for (int64 i = 0; i < index.size(); ++i) { + index[i] = (index[i] + 1) * tile_shape_.dimensions(i); + } + return index; +} + +StatusOr HloSharding::UniqueDevice() const { + if (!replicated_ && maximal_) { + return static_cast(*tile_assignment_.begin()); + } + return tensorflow::errors::InvalidArgument( + "UniqueDevice() called on sharding that executes on multiple devices"); +} + +Status HloSharding::Validate(const Shape& shape, int64 num_devices) const { + if (replicated_) { + return Status::OK(); + } + + // All tile assignments must be less than the number of available cores and + // unique. + Status status = Status::OK(); + std::set seen_cores; + tile_assignment_.Each( + [&](tensorflow::gtl::ArraySlice indices, uint32 core) { + // Don't overwrite a bad status, so we report the first error. + if (status.ok()) { + if (core >= num_devices) { + status = + tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "core ", core, " > ", num_devices, " in tile assignment")); + } else if (seen_cores.count(core) != 0) { + status = + tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "core ", core, " is not unique in tile assignment")); + } + } + seen_cores.insert(core); + }); + if (!status.ok()) { + return status; + } + + if (IsTileMaximal()) { + return Status::OK(); + } + + // The tile rank must be the same as the input rank. + if (ShapeUtil::Rank(shape) != ShapeUtil::Rank(tile_shape_)) { + return tensorflow::errors::InvalidArgument( + "Tile rank is different to the input rank"); + } + + // The tile shape must not be the same as the input shape without maximal_ + // also set. If this is the case, we're not actually sharded and the correct + // constructor should have been used. + if (ShapeUtil::Equal(shape, tile_shape_)) { + return tensorflow::errors::InvalidArgument( + "Tile shape is the same as the input shape. If a replicated sharding " + "was intended, use HloSharding::Replicated(). If a device placement " + "was intended, use HloSharding::AssignDevice()"); + } + + // The tile shape must not be greater than the input shape in any dimension. + for (int64 i = 0, e = ShapeUtil::Rank(shape); i != e; ++i) { + auto tile_dim = tile_shape_.dimensions(i); + auto shape_dim = shape.dimensions(i); + if (tile_dim > shape_dim) { + return tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "Tile is larger than input shape (dimension ", i, ", ", tile_dim, + " > ", shape_dim)); + } + } + + // The tile assignment tensor must be exactly dimensioned to ceil(shape[dim] + // tile[dim]) for every dimension contained within tile. + for (int64 i = 0, e = tile_assignment_.dimensions().size(); i != e; ++i) { + int64 expected_dim = + CeilOfRatio(shape.dimensions(i), tile_shape_.dimensions(i)); + if (tile_assignment_.dimensions()[i] != expected_dim) { + return tensorflow::errors::InvalidArgument(tensorflow::strings::StrCat( + "Tile assignment tensor has incorrect shape. Dimension ", i, + " expected ", expected_dim, " but got ", + tile_assignment_.dimensions()[i])); + } + } + + return Status::OK(); +} + +/*static*/ StatusOr HloSharding::FromProto( + const OpSharding& proto) { + if (proto.type() == OpSharding::Type::OpSharding_Type_REPLICATED) { + return Replicate(); + } else if (proto.type() == OpSharding::Type::OpSharding_Type_MAXIMAL) { + return HloSharding(proto.tile_assignment_devices(0)); + } + // Some versions of gcc cannot infer the TileAssignment constructor from a + // braced initializer-list, so create one manually. + std::vector devices(proto.tile_assignment_devices().begin(), + proto.tile_assignment_devices().end()); + Array tile_assignment( + std::vector(proto.tile_assignment_dimensions().begin(), + proto.tile_assignment_dimensions().end())); + std::copy(proto.tile_assignment_devices().begin(), + proto.tile_assignment_devices().end(), tile_assignment.begin()); + return HloSharding(proto.tile_shape(), tile_assignment); +} + +OpSharding HloSharding::ToProto() const { + OpSharding result; + *result.mutable_tile_shape() = tile_shape_; + for (int64 dim : tile_assignment_.dimensions()) { + result.add_tile_assignment_dimensions(dim); + } + for (auto device : tile_assignment_) { + result.add_tile_assignment_devices(device); + } + if (IsReplicated()) { + result.set_type(OpSharding::Type::OpSharding_Type_REPLICATED); + } else if (IsTileMaximal()) { + result.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + } else { + result.set_type(OpSharding::Type::OpSharding_Type_OTHER); + } + return result; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h new file mode 100644 index 0000000000000000000000000000000000000000..d7ada30c70bc3b41b3117375380eac2e883d9a9d --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_sharding.h @@ -0,0 +1,165 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// HLO shardings describe how an HLO instruction is split across multiple +// computations. + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_ + +#include + +#include "tensorflow/compiler/xla/array.h" +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/protobuf_util.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/gtl/array_slice.h" +#include "tensorflow/core/lib/hash/hash.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { + +// HLO shardings describe how an HLO instruction is split across multiple +// computations. +class HloSharding { + public: + // Creates a trivial sharding that replicates a maximal tile across all + // devices. + static HloSharding Replicate() { return HloSharding(); } + + // Creates a sharding that emulates device placement; a tile shape equal to + // the input shape (one tile) assigned to a single device. + static HloSharding AssignDevice(int64 device_id); + + // Creates a new sharding which splits a shape into tiles each with shape + // `tile_shape`. Each tile is assigned to one device, which is specified by + // `tile_assignment`. Any tensor not a multiple of the tile size in any + // dimension is implicitly padded to the tile size. + // + // e.g. Tile({2, 2}, {0, 1}) on a tensor of shape {3, 2} would look like: + // 2 1 padding + // <------><-> + // +----+----+ + // | 0 | 1 | + // +----+----+ + // + // Split into two tiles, one of which is implicitly padded by one. + static HloSharding Tile(const Shape& tile_shape, + const Array& tile_assignment) { + return HloSharding(tile_shape, tile_assignment); + } + + // Creates a new sharding which splits a one-dimensional input shape into + // `num_tiles` tiles. + static HloSharding Tile1D(const Shape& input_shape, int64 num_tiles); + + // Create a new sharding from a protobuf OpSharding. + static StatusOr FromProto(const OpSharding& proto); + + OpSharding ToProto() const; + string ToString() const; + + // Validate that this sharding can be applied to a tensor with shape `shape`. + Status Validate(const Shape& shape, int64 num_devices) const; + + // Returns true if the sharding is trivial: replicate on all devices. + bool IsReplicated() const { return replicated_; } + + // Returns true if the tile size is the same as the input size. + bool IsTileMaximal() const { return maximal_; } + + // Returns true if the sharding defines an operation on the given device. + bool UsesDevice(int64 device) const; + + // Returns the tile that should be executed on the given device. + std::vector TileIndexForDevice(int64 device) const; + + // Returns the device that should execute the given tile. + // It is an error to call this if is_replicated() is true. + int64 DeviceForTileIndex(tensorflow::gtl::ArraySlice index) const; + + // Given a device ID, returns the offset within the input space of the + // tile that should be executed on the given core. This returns the lower + // extent of the tile in the input space. + std::vector TileOffsetForDevice(int64 device) const; + + // Given a device ID, returns the limit within the input space of the + // tile that should be executed on the given core. This returns the upper + // extent of the tile in the input space. + std::vector TileLimitForDevice(int64 device) const; + + // Returns the single device this op operates on. + // Requires !Replicated() && IsTileMaximal(). + StatusOr UniqueDevice() const; + + // Returns true if this op only uses a single device. + bool HasUniqueDevice() const { return !IsReplicated() && IsTileMaximal(); } + + bool operator==(const HloSharding& other) const { + return replicated_ == other.replicated_ && maximal_ == other.maximal_ && + protobuf_util::ProtobufEquals(tile_shape_, other.tile_shape_) && + tile_assignment_ == other.tile_assignment_; + } + bool operator!=(const HloSharding& other) const { return !(*this == other); } + + size_t Hash() const { + if (replicated_) { + return 0; + } + size_t h = 0; + for (uint32 v : tile_assignment_) { + h = tensorflow::Hash64Combine(h, std::hash{}(v)); + } + for (uint32 v : tile_shape_.dimensions()) { + h = tensorflow::Hash64Combine(h, std::hash{}(v)); + } + return h; + } + + // Gets the tile shape. + // It is an error to call this if IsTileMaximal() is true. + const Shape& tile_shape() const { return tile_shape_; } + // Gets the tile assignment tensor. + // It is an error to call this if IsReplicated() is true. + const Array& tile_assignment() const { return tile_assignment_; } + + private: + HloSharding() + : replicated_(true), + maximal_(true), + tile_shape_(), + tile_assignment_({0}) {} + explicit HloSharding(int64 device_id) + : replicated_(false), + maximal_(true), + tile_shape_(), + tile_assignment_({1}, device_id) {} + HloSharding(const Shape& tile_shape, const Array& tile_assignment) + : replicated_(false), + maximal_(false), + tile_shape_(tile_shape), + tile_assignment_(tile_assignment) {} + + bool replicated_; + bool maximal_; + Shape tile_shape_; + Array tile_assignment_; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_ diff --git a/tensorflow/compiler/xla/service/hlo_sharding_test.cc b/tensorflow/compiler/xla/service/hlo_sharding_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..d0a20471a0f22a5fa414b71bb5160eed7cdc431b --- /dev/null +++ b/tensorflow/compiler/xla/service/hlo_sharding_test.cc @@ -0,0 +1,190 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/hlo_sharding.h" + +#include +#include +#include +#include + +#include "tensorflow/compiler/xla/literal_util.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/compiler/xla/test_helpers.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/util.h" + +namespace xla { +namespace { + +Array MakeArray(tensorflow::gtl::ArraySlice dimensions, + tensorflow::gtl::ArraySlice contents) { + Array a(dimensions); + std::copy(contents.begin(), contents.end(), a.begin()); + return a; +} + +class HloShardingTest : public HloTestBase {}; + +TEST_F(HloShardingTest, Replicate) { + Shape tile_shape = ShapeUtil::MakeShape(U32, {4}); + HloSharding sharding = HloSharding::Replicate(); + EXPECT_TRUE(sharding.IsReplicated()); + EXPECT_TRUE(sharding.IsTileMaximal()); + EXPECT_TRUE(sharding.UsesDevice(0)); + EXPECT_TRUE(sharding.UsesDevice(65535)); + + HloSharding other = HloSharding::Replicate(); + EXPECT_EQ(other, sharding); + + EXPECT_IS_OK(sharding.Validate(ShapeUtil::MakeShape(U32, {4}), + /*num_devices=*/2)); + EXPECT_IS_NOT_OK(sharding.UniqueDevice()); +} + +TEST_F(HloShardingTest, DevicePlacement) { + HloSharding sharding = HloSharding::AssignDevice(5); + EXPECT_FALSE(sharding.IsReplicated()); + EXPECT_TRUE(sharding.IsTileMaximal()); + EXPECT_FALSE(sharding.UsesDevice(0)); + EXPECT_TRUE(sharding.UsesDevice(5)); + EXPECT_EQ(5, sharding.UniqueDevice().ValueOrDie()); + + HloSharding other = HloSharding::Replicate(); + EXPECT_NE(other, sharding); + + EXPECT_IS_OK(sharding.Validate(ShapeUtil::MakeShape(U32, {4}), + /*num_devices=*/6)); + EXPECT_IS_NOT_OK( + sharding.Validate(ShapeUtil::MakeShape(U32, {4}), /*num_devices=*/5)); +} + +TEST_F(HloShardingTest, Tile) { + { + // Test should fail because of a duplicate tile assignment. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 0, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {4, 6}), + /*num_devices=*/4)); + } + + { + // Test should pass. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 1, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(U32, {4, 6}), + /*num_devices=*/2)); + } + + { + // Test should fail due to the tile being larger than the input space. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 1, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {2, 2}), + /*num_devices=*/4)); + } + + { + // Test should fail due to the tile not dividing the input space into 4 + // sections (even with padding). + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 1, 2, 3})); + EXPECT_IS_NOT_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {6, 3}), + /*num_devices=*/4)); + } + + { + // Test should pass. + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + EXPECT_IS_OK(sharding.Validate(ShapeUtil::MakeShape(F32, {3, 5}), + /*num_devices=*/5)); + + EXPECT_EQ(0, sharding.DeviceForTileIndex({0, 0})); + EXPECT_EQ(3, sharding.DeviceForTileIndex({0, 1})); + EXPECT_EQ(2, sharding.DeviceForTileIndex({1, 0})); + EXPECT_EQ(1, sharding.DeviceForTileIndex({1, 1})); + + EXPECT_EQ(sharding.TileOffsetForDevice(0), (std::vector{0, 0})); + EXPECT_EQ(sharding.TileOffsetForDevice(3), (std::vector{0, 3})); + EXPECT_EQ(sharding.TileOffsetForDevice(2), (std::vector{2, 0})); + EXPECT_EQ(sharding.TileOffsetForDevice(1), (std::vector{2, 3})); + + EXPECT_IS_NOT_OK(sharding.UniqueDevice()); + } +} + +TEST_F(HloShardingTest, Hash) { + auto hash_compare_equal = [](const HloSharding& a, const HloSharding& b) { + if (a.Hash() != b.Hash()) { + return false; + } + return a == b; + }; + + { + HloSharding sharding1 = HloSharding::Replicate(); + HloSharding sharding2 = HloSharding::Replicate(); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + HloSharding sharding1 = HloSharding::AssignDevice(1); + HloSharding sharding2 = HloSharding::AssignDevice(1); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + HloSharding sharding1 = HloSharding::AssignDevice(1); + HloSharding sharding2 = HloSharding::AssignDevice(2); + EXPECT_FALSE(hash_compare_equal(sharding1, sharding2)); + } + + { + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding1 = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + HloSharding sharding2 = HloSharding::Tile(ShapeUtil::MakeShape(U32, {2, 3}), + MakeArray({2, 2}, {0, 3, 2, 1})); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding1 = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + HloSharding sharding2 = HloSharding::Tile(ShapeUtil::MakeShape(U32, {2, 3}), + MakeArray({2, 2}, {0, 3, 2, 1})); + EXPECT_TRUE(hash_compare_equal(sharding1, sharding2)); + } + + { + Shape tile_shape = ShapeUtil::MakeShape(U32, {2, 3}); + HloSharding sharding1 = + HloSharding::Tile(tile_shape, MakeArray({2, 2}, {0, 3, 2, 1})); + HloSharding sharding2 = HloSharding::Tile(ShapeUtil::MakeShape(U32, {2, 3}), + MakeArray({2, 2}, {0, 3, 1, 2})); + EXPECT_FALSE(hash_compare_equal(sharding1, sharding2)); + } +} + +} // namespace +} // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc index 3f6d89f24f4ec76d913611d03dd28b93a09d34a1..06abe007477dbcd00bcdc7f2656c4dece6d1cf74 100644 --- a/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc +++ b/tensorflow/compiler/xla/service/hlo_tfgraph_builder.cc @@ -56,6 +56,8 @@ TensorShapeProto GetTensorShape(const HloInstruction* instruction) { return tensor_shape; } +string GetDeviceName(int device) { return StrCat("/device/XLA:", device); } + } // namespace void CleanNodeName(string* name) { @@ -178,6 +180,10 @@ void HloTfGraphBuilder::SetNodeAttrs(const HloInstruction* instruction, case HloOpcode::kCustomCall: attrs["custom_call_target"].set_s(instruction->custom_call_target()); break; + case HloOpcode::kSend: + case HloOpcode::kRecv: + attrs["channel_id"].set_i(instruction->channel_id()); + break; default: break; } @@ -192,6 +198,11 @@ Status HloTfGraphBuilder::AddInstruction(const HloInstruction* instruction) { NodeDef* node_def = graph_def_.add_node(); node_def->set_name(GetNodeNameForInstruction(instruction)); node_def->set_op(GetOpDefName(instruction)); + if (instruction->has_sharding() && + instruction->sharding().HasUniqueDevice()) { + TF_ASSIGN_OR_RETURN(int64 device, instruction->sharding().UniqueDevice()); + node_def->set_device(GetDeviceName(device)); + } SetNodeAttrs(instruction, node_def); if (instruction->opcode() == HloOpcode::kFusion) { for (auto* fused_instruction : instruction->fused_instructions()) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 35dff4a957f023d0f34082d7db1b6a6ade9c15f8..c1aa655401a2be68af943e2ed29c4ab99d341383 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -40,22 +40,17 @@ class ShapeVerifier : public DfsHloVisitor { return CheckBinaryShape(hlo); } - Status HandleClamp(HloInstruction* clamp, HloInstruction* min, - HloInstruction* arg, HloInstruction* max) override { + Status HandleClamp(HloInstruction* clamp) override { return CheckTernaryShape(clamp); } - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override { + Status HandleSelect(HloInstruction* select) override { return CheckTernaryShape(select); } - Status HandleConcatenate( - HloInstruction* concatenate, - tensorflow::gtl::ArraySlice operands) override { + Status HandleConcatenate(HloInstruction* concatenate) override { std::vector operand_shapes; - for (const HloInstruction* operand : operands) { + for (const HloInstruction* operand : concatenate->operands()) { operand_shapes.push_back(&operand->shape()); } return CheckShape( @@ -64,6 +59,10 @@ class ShapeVerifier : public DfsHloVisitor { } Status HandleConvert(HloInstruction* convert) override { + if (ShapeUtil::ElementIsComplex(convert->operand(0)->shape())) { + TF_RET_CHECK(ShapeUtil::ElementIsComplex(convert->shape())) + << "Unsupported complex->real kConvert"; + } return CheckShape(convert, ShapeInference::InferConvertShape( convert->operand(0)->shape(), convert->shape().element_type())); @@ -73,17 +72,17 @@ class ShapeVerifier : public DfsHloVisitor { return CheckUnaryShape(copy); } - Status HandleDot(HloInstruction* dot, HloInstruction* lhs, - HloInstruction* rhs) override { + Status HandleDot(HloInstruction* dot) override { return CheckBinaryShape(dot); } - Status HandleConvolution(HloInstruction* convolution, HloInstruction* lhs, - HloInstruction* rhs, const Window& window) override { - TF_ASSIGN_OR_RETURN(const Shape expected, - ShapeInference::InferConvolveShape( - lhs->shape(), rhs->shape(), window, - convolution->convolution_dimension_numbers())); + Status HandleConvolution(HloInstruction* convolution) override { + TF_ASSIGN_OR_RETURN( + const Shape expected, + ShapeInference::InferConvolveShape( + convolution->operand(0)->shape(), convolution->operand(1)->shape(), + convolution->window(), + convolution->convolution_dimension_numbers())); return CheckShape(convolution, expected); } @@ -100,47 +99,40 @@ class ShapeVerifier : public DfsHloVisitor { reduce_precision->mantissa_bits())); } - Status HandleInfeed(HloInstruction* infeed) override { + Status HandleInfeed(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleOutfeed(HloInstruction* outfeed) override { + Status HandleOutfeed(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleRng(HloInstruction* random, - RandomDistribution distribution) override { + Status HandleRng(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleReverse(HloInstruction* reverse, - HloInstruction* operand) override { + Status HandleReverse(HloInstruction* reverse) override { return CheckShape( reverse, ShapeInference::InferReverseShape(reverse->operand(0)->shape(), reverse->dimensions())); } - Status HandleSort(HloInstruction* sort, HloInstruction* operand) override { + Status HandleSort(HloInstruction* sort) override { return CheckUnaryShape(sort); } - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override { - return CheckShape(constant, literal.shape()); + Status HandleConstant(HloInstruction* constant) override { + return CheckShape(constant, constant->literal().shape()); } - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override { + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override { return CheckShape(get_tuple_element, ShapeInference::InferGetTupleElementShape( get_tuple_element->operand(0)->shape(), get_tuple_element->tuple_index())); } - Status HandleReduce(HloInstruction* reduce, HloInstruction* arg, - HloInstruction* init_value, - tensorflow::gtl::ArraySlice dimensions, - HloComputation* function) override { + Status HandleReduce(HloInstruction* reduce) override { return CheckShape( reduce, ShapeInference::InferReduceShape( @@ -183,11 +175,11 @@ class ShapeVerifier : public DfsHloVisitor { transpose->dimensions())); } - Status HandleParameter(HloInstruction* parameter) override { + Status HandleParameter(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleFusion(HloInstruction* fusion) override { + Status HandleFusion(HloInstruction*) override { return tensorflow::Status::OK(); } @@ -196,32 +188,26 @@ class ShapeVerifier : public DfsHloVisitor { return CheckShape(call, call->to_apply()->ComputeProgramShape().result()); } - Status HandleCustomCall(HloInstruction* custom_call, - tensorflow::gtl::ArraySlice operands, - tensorflow::StringPiece custom_call_target) override { + Status HandleCustomCall(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleSlice(HloInstruction* slice, HloInstruction* operand) override { + Status HandleSlice(HloInstruction* slice) override { return CheckShape(slice, ShapeInference::InferSliceShape( slice->operand(0)->shape(), slice->slice_starts(), slice->slice_limits(), slice->slice_strides())); } - Status HandleDynamicSlice(HloInstruction* dynamic_slice, - HloInstruction* operand, - HloInstruction* start_indices) override { + Status HandleDynamicSlice(HloInstruction* dynamic_slice) override { return CheckShape(dynamic_slice, ShapeInference::InferDynamicSliceShape( dynamic_slice->operand(0)->shape(), dynamic_slice->operand(1)->shape(), dynamic_slice->dynamic_slice_sizes())); } - Status HandleDynamicUpdateSlice(HloInstruction* dynamic_update_slice, - HloInstruction* operand, - HloInstruction* update, - HloInstruction* start_indices) override { + Status HandleDynamicUpdateSlice( + HloInstruction* dynamic_update_slice) override { return CheckShape(dynamic_update_slice, ShapeInference::InferDynamicUpdateSliceShape( dynamic_update_slice->operand(0)->shape(), @@ -229,20 +215,14 @@ class ShapeVerifier : public DfsHloVisitor { dynamic_update_slice->operand(2)->shape())); } - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override { + Status HandleTuple(HloInstruction* tuple) override { return CheckVariadicShape(tuple); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override { + Status HandleMap(HloInstruction* map) override { std::vector operand_shapes; int64 max_operand_rank = 0; - for (const HloInstruction* operand : operands) { + for (const HloInstruction* operand : map->operands()) { operand_shapes.push_back(&operand->shape()); max_operand_rank = std::max(max_operand_rank, ShapeUtil::Rank(operand->shape())); @@ -257,9 +237,7 @@ class ShapeVerifier : public DfsHloVisitor { operand_shapes, map->to_apply()->ComputeProgramShape(), map_dims)); } - Status HandleReduceWindow(HloInstruction* reduce_window, - HloInstruction* operand, const Window& window, - HloComputation* function) override { + Status HandleReduceWindow(HloInstruction* reduce_window) override { return CheckShape( reduce_window, ShapeInference::InferReduceWindowShape( @@ -292,11 +270,11 @@ class ShapeVerifier : public DfsHloVisitor { pad->padding_config())); } - Status HandleSend(HloInstruction* send) override { + Status HandleSend(HloInstruction*) override { return tensorflow::Status::OK(); } - Status HandleRecv(HloInstruction* recv) override { + Status HandleRecv(HloInstruction*) override { return tensorflow::Status::OK(); } @@ -331,7 +309,7 @@ class ShapeVerifier : public DfsHloVisitor { batch_norm_grad->feature_index())); } - Status FinishVisit(HloInstruction* root) override { + Status FinishVisit(HloInstruction*) override { return tensorflow::Status::OK(); } @@ -355,7 +333,10 @@ class ShapeVerifier : public DfsHloVisitor { Status CheckShape(const HloInstruction* instruction, const StatusOr& expected_shape_status) { if (!expected_shape_status.ok()) { - return expected_shape_status.status(); + Status s = expected_shape_status.status(); + tensorflow::errors::AppendToMessage(&s, ", for instruction ", + instruction->ToString()); + return s; } return CheckShape(instruction, expected_shape_status.ValueOrDie()); } diff --git a/tensorflow/compiler/xla/service/inliner.cc b/tensorflow/compiler/xla/service/inliner.cc index 6ea0f127d53404af9514820b36a97bb0526aa5f9..5c193fceb984448cf0532d7e1010281268614293 100644 --- a/tensorflow/compiler/xla/service/inliner.cc +++ b/tensorflow/compiler/xla/service/inliner.cc @@ -43,11 +43,7 @@ class InlinerVisitor : public DfsHloVisitorWithDefault { return Status::OK(); } - Status HandleMap( - HloInstruction* map, - tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice static_operands) override; + Status HandleMap(HloInstruction* map) override; // Runs the visitor on a computation. StatusOr Run(HloComputation* computation); @@ -67,18 +63,14 @@ StatusOr InlinerVisitor::Run(HloComputation* computation) { return changed_; } -Status InlinerVisitor::HandleMap( - HloInstruction* map, tensorflow::gtl::ArraySlice operands, - HloComputation* function, - tensorflow::gtl::ArraySlice /*static_operands*/) { +Status InlinerVisitor::HandleMap(HloInstruction* map) { + HloComputation* function = map->to_apply(); HloInstruction& root = *function->root_instruction(); // TODO(b/29249531): Add DCE pass to remove unused HloComputations. // Only inlining functions that are simply a single operation until a better // profitability model for inlining is defined. if (hlo_query::AllOperandsAreParameters(root)) { - if (root.opcode() == HloOpcode::kUpdate || - root.opcode() == HloOpcode::kFusion || - root.opcode() == HloOpcode::kIndex || + if (root.opcode() == HloOpcode::kFusion || root.opcode() == HloOpcode::kParameter || root.opcode() == HloOpcode::kTrace) { // Cloning not supported for these instructions. @@ -92,7 +84,7 @@ Status InlinerVisitor::HandleMap( if (root.opcode() != HloOpcode::kConstant) { std::vector params; for (int64 o = 0; o < root.operands().size(); o++) { - params.push_back(operands[root.operand(o)->parameter_number()]); + params.push_back(map->operands()[root.operand(o)->parameter_number()]); } HloInstruction* placed_instruction = computation_->AddInstruction( root.CloneWithNewOperands(map->shape(), params)); diff --git a/tensorflow/compiler/xla/service/instruction_fusion.cc b/tensorflow/compiler/xla/service/instruction_fusion.cc index 7e46d79ba41cc27894de892c100d5e71eb3153f1..0d1b7bc109c56bc4290ede09284c6d20142bdb08 100644 --- a/tensorflow/compiler/xla/service/instruction_fusion.cc +++ b/tensorflow/compiler/xla/service/instruction_fusion.cc @@ -32,17 +32,16 @@ namespace xla { const HloInstruction& instruction) { switch (instruction.opcode()) { // Cheap instructions. - case HloOpcode::kAbs: case HloOpcode::kAdd: case HloOpcode::kBitcast: case HloOpcode::kBroadcast: case HloOpcode::kCeil: case HloOpcode::kClamp: + case HloOpcode::kComplex: case HloOpcode::kConcatenate: case HloOpcode::kConstant: case HloOpcode::kConvert: case HloOpcode::kCopy: - case HloOpcode::kCos: case HloOpcode::kDynamicSlice: case HloOpcode::kDynamicUpdateSlice: case HloOpcode::kEq: @@ -50,6 +49,7 @@ namespace xla { case HloOpcode::kGe: case HloOpcode::kGetTupleElement: case HloOpcode::kGt: + case HloOpcode::kImag: case HloOpcode::kInfeed: case HloOpcode::kIsFinite: case HloOpcode::kLe: @@ -64,6 +64,7 @@ namespace xla { case HloOpcode::kNegate: case HloOpcode::kOutfeed: case HloOpcode::kPad: + case HloOpcode::kReal: case HloOpcode::kReducePrecision: case HloOpcode::kReshape: case HloOpcode::kReverse: @@ -72,15 +73,21 @@ namespace xla { case HloOpcode::kShiftLeft: case HloOpcode::kShiftRightArithmetic: case HloOpcode::kShiftRightLogical: - case HloOpcode::kSign: - case HloOpcode::kSin: case HloOpcode::kSlice: case HloOpcode::kSubtract: case HloOpcode::kTranspose: case HloOpcode::kTuple: return false; + // Cheap instructions for reals, but expensive for complex. + case HloOpcode::kAbs: + case HloOpcode::kCos: + case HloOpcode::kSign: + case HloOpcode::kSin: + return ShapeUtil::ElementIsComplex(instruction.shape()); + // Expensive instructions. + case HloOpcode::kAtan2: case HloOpcode::kBatchNormTraining: case HloOpcode::kBatchNormInference: case HloOpcode::kBatchNormGrad: @@ -92,7 +99,6 @@ namespace xla { case HloOpcode::kDot: case HloOpcode::kExp: case HloOpcode::kFusion: - case HloOpcode::kIndex: case HloOpcode::kLog: case HloOpcode::kMap: case HloOpcode::kParameter: @@ -105,7 +111,6 @@ namespace xla { case HloOpcode::kSort: case HloOpcode::kTanh: case HloOpcode::kTrace: - case HloOpcode::kUpdate: case HloOpcode::kWhile: case HloOpcode::kSend: case HloOpcode::kRecv: diff --git a/tensorflow/compiler/xla/service/layout_assignment.cc b/tensorflow/compiler/xla/service/layout_assignment.cc index 2058706f1120238f63c06c8dcac79b8487888df5..7eda7c2284c2457703fcfcd4226172e41dd4ae01 100644 --- a/tensorflow/compiler/xla/service/layout_assignment.cc +++ b/tensorflow/compiler/xla/service/layout_assignment.cc @@ -732,7 +732,8 @@ std::unique_ptr LayoutAssignment::ChooseOperandLayoutFromOutputLayout( // dimension bound is 1 in the operand shape, there may be several such // layouts. So if 'output_layout' is the default layout, try if the // reshape is a bitcast when using the same layout. This may avoid copy - // operations. + // operations. For similar reasons, if the operand and output have the same + // rank, try to match the operand's layout to the output. if (ShapeUtil::TrueRank(operand->shape()) == 1 && ShapeUtil::Rank(instruction->shape()) == 1) { // Don't assign a layout in case of R1 -> effective R1 reshape. @@ -748,6 +749,13 @@ std::unique_ptr LayoutAssignment::ChooseOperandLayoutFromOutputLayout( if (ShapeUtil::ReshapeIsBitcast(operand_shape, output_shape_with_layout)) { return MakeUnique(operand_shape.layout()); } + if (ShapeUtil::Rank(operand_shape) == ShapeUtil::Rank(output_shape)) { + *operand_shape.mutable_layout() = output_layout; + if (ShapeUtil::ReshapeIsBitcast(operand_shape, + output_shape_with_layout)) { + return MakeUnique(output_layout); + } + } auto aligned_operand_shape = ShapeUtil::AlignLayouts(output_shape_with_layout, operand_shape); if (aligned_operand_shape) { @@ -796,7 +804,8 @@ std::unique_ptr LayoutAssignment::ChooseOutputLayoutFromOperandLayout( // dimension bound is 1 in the user shape, there may be several such // layouts. So if 'operand_layout' is the default layout, try if the // reshape is a bitcast when using the same layout. This may avoid copy - // operations. + // operations. For similar reasons, if the operand and output have the same + // rank, try to match the outputs's layout to the operand. if (ShapeUtil::Rank(operand->shape()) == 1 && ShapeUtil::TrueRank(user->shape()) == 1) { // Don't assign a layout in case of R1 -> effective R1 reshape. @@ -812,6 +821,13 @@ std::unique_ptr LayoutAssignment::ChooseOutputLayoutFromOperandLayout( if (ShapeUtil::ReshapeIsBitcast(output_shape, operand_shape_with_layout)) { return MakeUnique(output_shape.layout()); } + if (ShapeUtil::Rank(operand->shape()) == ShapeUtil::Rank(output_shape)) { + *output_shape.mutable_layout() = operand_layout; + if (ShapeUtil::ReshapeIsBitcast(output_shape, + operand_shape_with_layout)) { + return MakeUnique(operand_layout); + } + } auto aligned_user_shape = ShapeUtil::AlignLayouts(operand_shape_with_layout, output_shape); if (aligned_user_shape) { diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD index 70579e3273394f0b5fa5e9d7863b291171b0bfa1..075d4a1ab5e5f39394ade393d21525ca3e97136e 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/BUILD +++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD @@ -137,7 +137,6 @@ cc_library( "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service/gpu:parallel_loop_emitter", "//tensorflow/compiler/xla/service/gpu:partition_assignment", - "@llvm//:core", ], ) diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc index d286c49d6868c91026c8901b7871a322dabd38ec..bc683a1880b010d57e83aa6e9ffa95fda299e1a0 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.cc @@ -72,10 +72,10 @@ Status FusedIrEmitter::DefaultAction(HloInstruction* hlo) { return Status::OK(); } -Status FusedIrEmitter::HandleConstant(HloInstruction* constant, - const Literal& literal) { +Status FusedIrEmitter::HandleConstant(HloInstruction* constant) { + const Literal& literal = constant->literal(); llvm::Constant* initializer = - llvm_ir::ConvertLiteralToIrConstant(literal, ir_builder_); + llvm_ir::ConvertLiteralToIrConstant(literal, module_); llvm::GlobalVariable* global = new llvm::GlobalVariable( *ir_builder_->GetInsertBlock()->getModule(), initializer->getType(), /*isConstant=*/true, llvm::GlobalValue::ExternalLinkage, initializer, @@ -88,9 +88,10 @@ Status FusedIrEmitter::HandleConstant(HloInstruction* constant, return Status::OK(); } -Status FusedIrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) { +Status FusedIrEmitter::HandleGetTupleElement( + HloInstruction* get_tuple_element) { // Lookup ir value for 'operand'. + auto operand = get_tuple_element->operand(0); auto it = gte_values_.find(operand); if (it == gte_values_.end()) { return Unimplemented( @@ -101,7 +102,7 @@ Status FusedIrEmitter::HandleGetTupleElement(HloInstruction* get_tuple_element, // Emit code to lookup tuple element pointer, and store it in 'gte_values_'. llvm::Value* tuple_element_ptr = llvm_ir::EmitGetTupleElement( get_tuple_element->shape(), get_tuple_element->tuple_index(), - /*alignment=*/1, it->second, ir_builder_); + /*alignment=*/1, it->second, ir_builder_, module_); gte_values_.insert(std::make_pair(get_tuple_element, tuple_element_ptr)); // Emit code to read base tuple element array (if non-tuple shaped). if (!ShapeUtil::IsTuple(get_tuple_element->shape())) { @@ -128,13 +129,12 @@ Status FusedIrEmitter::HandleParameter(HloInstruction* parameter) { return Status::OK(); } -Status FusedIrEmitter::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status FusedIrEmitter::HandleTuple(HloInstruction* tuple) { + tensorflow::gtl::ArraySlice operands(tuple->operands()); std::vector operand_elemental_ir_types; for (HloInstruction* operand : operands) { operand_elemental_ir_types.push_back(llvm_ir::PrimitiveTypeToIrType( - operand->shape().element_type(), ir_builder_)); + operand->shape().element_type(), module_)); } generators_[tuple] = [=](const IrArray::Index& index) -> StatusOr { diff --git a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h index a24e104067f19e45ab2566beedbb8217913bad12..9ad7cd82cb8ca862fd7acec3dfb12c9fd61f6e27 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h +++ b/tensorflow/compiler/xla/service/llvm_ir/fused_ir_emitter.h @@ -42,22 +42,19 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault { ElementalIrEmitter* elemental_emitter) : parameter_arrays_(parameter_arrays), elemental_emitter_(elemental_emitter), - ir_builder_(elemental_emitter->ir_builder()) {} + ir_builder_(elemental_emitter->ir_builder()), + module_(elemental_emitter->module()) {} Status DefaultAction(HloInstruction* hlo) override; - Status HandleConstant(HloInstruction* constant, - const Literal& literal) override; + Status HandleConstant(HloInstruction* constant) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleParameter(HloInstruction* parameter) override; // Emits the ir value for each element in the tuple. - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; + Status HandleTuple(HloInstruction* tuple) override; Status FinishVisit(HloInstruction* root) override; @@ -85,6 +82,7 @@ class FusedIrEmitter : public DfsHloVisitorWithDefault { // Borrowed llvm::IRBuilder<>* ir_builder_; + llvm::Module* module_; // Map from instruction pointers to functions to generate elements of their // outputs diff --git a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc index 6a00a565c6d23aa8cd5f4e17621de8ca99dd1c5d..e3f98ac13e76f0df465066422ca7918a0f218b60 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/ir_array.cc @@ -229,9 +229,11 @@ llvm::Value* IrArray::EmitArrayElementAddress( } if (!is_implicit_broadcast && index.LinearValidOnShape(*shape_)) { + llvm::Module* module = + ir_builder->GetInsertBlock()->getParent()->getParent(); return ir_builder->CreateInBoundsGEP( ir_builder->CreateBitCast( - base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), ir_builder) + base_ptr_, PrimitiveTypeToIrType(shape_->element_type(), module) ->getPointerTo()), {index.linear()}, llvm_ir::AsStringRef(name)); } @@ -281,7 +283,8 @@ void IrArray::EmitWriteArrayElement(const Index& index, llvm::Value* value, IrArray IrArray::CastToShape(const Shape& new_shape, llvm::IRBuilder<>* ir_builder) const { - llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, ir_builder); + llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent(); + llvm::Type* new_ir_type = llvm_ir::ShapeToIrType(new_shape, module); return IrArray( ir_builder->CreatePointerCast(base_ptr_, new_ir_type->getPointerTo()), new_shape); diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index 8e188e7ae848b093abb2f7ba84b36413d397f7c8..5dff4b5778970dd473c5f158b3828a850847d1ff 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Operator.h" #include "llvm/Target/TargetOptions.h" @@ -38,6 +39,19 @@ limitations under the License. namespace xla { namespace llvm_ir { +namespace { + +// Note, this function is only useful in an insertion context; in a global +// (e.g. constants) context it will CHECK fail. +llvm::Module* ModuleFromIRBuilder(llvm::IRBuilder<>* ir_builder) { + auto block = CHECK_NOTNULL(ir_builder->GetInsertBlock()); + auto fn = CHECK_NOTNULL(block->getParent()); + auto module = CHECK_NOTNULL(fn->getParent()); + return module; +} + +} // namespace + string AsString(const std::string& str) { return string(str.data(), str.length()); } @@ -63,7 +77,7 @@ llvm::Value* EmitCallToIntrinsic( for (auto type : overloaded_types) { types.push_back(type); } - llvm::Module* module = ir_builder->GetInsertBlock()->getParent()->getParent(); + llvm::Module* module = ModuleFromIRBuilder(ir_builder); llvm::Function* intrinsic = llvm::Intrinsic::getDeclaration(module, intrinsic_id, types); std::vector operands_vec; @@ -119,38 +133,53 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index, } llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type, - llvm::IRBuilder<>* ir_builder) { + llvm::Module* module) { switch (element_type) { case PRED: case S8: case U8: - return ir_builder->getInt8Ty(); + return llvm::Type::getInt8Ty(module->getContext()); case S16: case U16: - return ir_builder->getInt16Ty(); + return llvm::Type::getInt16Ty(module->getContext()); case S32: case U32: - return ir_builder->getInt32Ty(); + return llvm::Type::getInt32Ty(module->getContext()); case S64: case U64: - return ir_builder->getInt64Ty(); + return llvm::Type::getInt64Ty(module->getContext()); case F32: - return ir_builder->getFloatTy(); + return llvm::Type::getFloatTy(module->getContext()); case F64: - return ir_builder->getDoubleTy(); + return llvm::Type::getDoubleTy(module->getContext()); + case C64: { + auto cplx_t = module->getTypeByName("complex64"); + if (cplx_t == nullptr) { + // C++ standard dictates the memory layout of std::complex is contiguous + // real followed by imaginary. C++11 section 26.4 [complex.numbers]: + // If z is an lvalue expression of type cv std::complex then the + // expression reinterpret_cast(z) shall be well-formed, + // reinterpret_cast(z)[0] shall designate the real part of + // z, and reinterpret_cast(z)[1] shall designate the + // imaginary part of z. + return llvm::StructType::create( + "complex64", llvm::Type::getFloatTy(module->getContext()), + llvm::Type::getFloatTy(module->getContext())); + } + return cplx_t; + } // A Tuple contains an array of pointers. Use i8*. case TUPLE: // An Opaque is like a void*, use i8*. case OPAQUE: - return ir_builder->getInt8PtrTy(); + return llvm::Type::getInt8PtrTy(module->getContext()); default: LOG(FATAL) << "unsupported type " << element_type; } } -llvm::Type* ShapeToIrType(const Shape& shape, llvm::IRBuilder<>* ir_builder) { - llvm::Type* result_type = - PrimitiveTypeToIrType(shape.element_type(), ir_builder); +llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module) { + llvm::Type* result_type = PrimitiveTypeToIrType(shape.element_type(), module); if (ShapeUtil::IsTuple(shape)) { // A tuple buffer is an array of pointers. result_type = llvm::ArrayType::get(result_type, shape.tuple_shapes_size()); @@ -197,10 +226,10 @@ namespace { // value down to zero). llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, std::vector* multi_index, - llvm::IRBuilder<>* ir_builder) { + llvm::Module* module) { const Shape& shape = literal.shape(); llvm::Type* ir_element_type = - llvm_ir::PrimitiveTypeToIrType(shape.element_type(), ir_builder); + llvm_ir::PrimitiveTypeToIrType(shape.element_type(), module); if (dimension_index == -1) { // Base case of the recursion. Index into the data field of the protobuf // with the multi index. @@ -238,6 +267,16 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, value = llvm::ConstantFP::get(ir_element_type, literal.Get(*multi_index)); break; + case C64: { + complex64 x = literal.Get(*multi_index); + value = llvm::ConstantStruct::get( + static_cast(ir_element_type), + llvm::ConstantFP::get(llvm_ir::PrimitiveTypeToIrType(F32, module), + x.real()), + llvm::ConstantFP::get(llvm_ir::PrimitiveTypeToIrType(F32, module), + x.imag())); + break; + } default: LOG(FATAL) << "unsupported type " << shape.element_type(); } @@ -256,8 +295,8 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, std::vector elements; for (int64 i = 0; i < shape.dimensions(dimension); ++i) { (*multi_index)[dimension] = i; - elements.push_back(LiteralToConstant(literal, dimension_index - 1, - multi_index, ir_builder)); + elements.push_back( + LiteralToConstant(literal, dimension_index - 1, multi_index, module)); } llvm::Type* element_type; @@ -279,11 +318,11 @@ llvm::Constant* LiteralToConstant(const Literal& literal, int64 dimension_index, } // namespace llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal, - llvm::IRBuilder<>* ir_builder) { + llvm::Module* module) { std::vector multi_index(ShapeUtil::Rank(literal.shape()), 0); llvm::Constant* value = LiteralToConstant( literal, /*dimension_index=*/ShapeUtil::Rank(literal.shape()) - 1, - &multi_index, ir_builder); + &multi_index, module); return value; } @@ -380,7 +419,8 @@ llvm::Value* EmitComparison(llvm::CmpInst::Predicate predicate, // comparison_result is i1, but the NVPTX codegen incorrectly lowers i1 // arrays. So we extend it to i8 so that it's addressable. return ir_builder->CreateZExt( - comparison_result, llvm_ir::PrimitiveTypeToIrType(PRED, ir_builder)); + comparison_result, + llvm_ir::PrimitiveTypeToIrType(PRED, ModuleFromIRBuilder(ir_builder))); } // Internal helper that is called from emitted code to log an int64 value with a diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h index 7a7d14da1eb62ab3d6401d2eff64c301c93a3806..304192b58e9331c2544f973bf65299111122aea8 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h @@ -127,11 +127,11 @@ llvm::Value* EmitBufferIndexingGEP(llvm::Value* array, int64 index, // Returns the LLVM type which represents the given XLA primitive type. llvm::Type* PrimitiveTypeToIrType(PrimitiveType element_type, - llvm::IRBuilder<>* ir_builder); + llvm::Module* module); // Returns the LLVM type which represents the given XLA shape. For example, // if "shape" is [5 x [10 x f32]], the function returns [5 x [10 x float]]. -llvm::Type* ShapeToIrType(const Shape& shape, llvm::IRBuilder<>* ir_builder); +llvm::Type* ShapeToIrType(const Shape& shape, llvm::Module* module); // Returns a value that represents a pointer to a global string constant that // encodes the shape as a serialized protobuf. @@ -149,7 +149,7 @@ StatusOr DecodeSelfDescribingShapeConstant(const void* shape_ptr, // Converts a given literal to an IR Constant. Literals have known constant // values at IR emission time. llvm::Constant* ConvertLiteralToIrConstant(const Literal& literal, - llvm::IRBuilder<>* ir_builder); + llvm::Module* module); // Inserts an allocate of the requested type at the entry point of the // function that the builder is currently building. The insert point diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc index 6051cbfc6f6a2d3cc99740beda5dee03a9392bdd..3a21eda35757aa706565ee4a5286eee1acea117b 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.cc @@ -31,14 +31,15 @@ namespace xla { namespace llvm_ir { void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true, - llvm::Value* on_false, llvm::IRBuilder<>* ir_builder) { + llvm::Value* on_false, llvm::IRBuilder<>* ir_builder, + llvm::Module* module) { CHECK(ShapeUtil::IsScalar(pred.GetShape())); llvm::LoadInst* pred_value = ir_builder->CreateLoad(pred.GetBasePointer(), "load_predicate_value"); llvm::Value* pred_cond = ir_builder->CreateICmpNE( pred_value, - llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, ir_builder), 0), + llvm::ConstantInt::get(PrimitiveTypeToIrType(PRED, module), 0), "boolean_predicate"); VLOG(2) << "HandleSelect for tuple:"; @@ -71,11 +72,11 @@ void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true, void EmitTuple(IrArray tuple, tensorflow::gtl::ArraySlice operands, - llvm::IRBuilder<>* ir_builder) { + llvm::IRBuilder<>* ir_builder, llvm::Module* module) { for (size_t i = 0; i < operands.size(); ++i) { auto* store = ir_builder->CreateStore( ir_builder->CreatePointerCast(operands[i], - PrimitiveTypeToIrType(TUPLE, ir_builder)), + PrimitiveTypeToIrType(TUPLE, module)), ir_builder->CreateInBoundsGEP( tuple.GetBasePointer(), {ir_builder->getInt64(0), ir_builder->getInt64(i)})); @@ -85,7 +86,8 @@ void EmitTuple(IrArray tuple, llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index, int alignment, llvm::Value* operand, - llvm::IRBuilder<>* ir_builder) { + llvm::IRBuilder<>* ir_builder, + llvm::Module* module) { llvm::Value* element_ptr = ir_builder->CreateInBoundsGEP( operand, {ir_builder->getInt64(0), ir_builder->getInt64(index)}); llvm::LoadInst* src_buffer = ir_builder->CreateLoad(element_ptr); @@ -98,7 +100,7 @@ llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index, } SetAlignmentMetadataForLoad(src_buffer, alignment); - llvm::Type* element_type = ShapeToIrType(target_shape, ir_builder); + llvm::Type* element_type = ShapeToIrType(target_shape, module); llvm::Value* ret_val = ir_builder->CreateBitCast(src_buffer, element_type->getPointerTo()); return ret_val; diff --git a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h index a75cdc815808fc3b9e8669dde1eddf995080f53d..dbf9a140068b60505f6798360438f709bfd3feba 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h +++ b/tensorflow/compiler/xla/service/llvm_ir/tuple_ops.h @@ -60,13 +60,14 @@ namespace llvm_ir { // tuple_on_true or tuple_on_false: // output[i] = pred ? tuple_on_true[i] : tuple_on_false[i] void EmitTupleSelect(IrArray select, IrArray pred, llvm::Value* on_true, - llvm::Value* on_false, llvm::IRBuilder<>* ir_builder); + llvm::Value* on_false, llvm::IRBuilder<>* ir_builder, + llvm::Module* module); // A tuple is an array of pointers, one for each operand. Each pointer points to // the output buffer of its corresponding operand. void EmitTuple(IrArray tuple, tensorflow::gtl::ArraySlice operands, - llvm::IRBuilder<>* ir_builder); + llvm::IRBuilder<>* ir_builder, llvm::Module* module); // A tuple is an array of pointers, one for each operand. Each pointer points to // the output buffer of its corresponding operand. A GetTupleElement instruction @@ -74,7 +75,8 @@ void EmitTuple(IrArray tuple, // Returns an llvm value representing a pointer to the tuple element buffer. llvm::Value* EmitGetTupleElement(const Shape& target_shape, int64 index, int alignment, llvm::Value* operand, - llvm::IRBuilder<>* ir_builder); + llvm::IRBuilder<>* ir_builder, + llvm::Module* module); } // namespace llvm_ir } // namespace xla diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc index bf3bb2ddf07aa29a71bad0c390e72e41ce8abbac..b92017c6cbc43d78ab4e5b32f25f5980b8d4ae56 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.cc +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.cc @@ -86,8 +86,7 @@ Status LogicalBufferAnalysis::DefaultAction(HloInstruction* hlo_instruction) { return Status::OK(); } -Status LogicalBufferAnalysis::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { +Status LogicalBufferAnalysis::HandleGetTupleElement(HloInstruction*) { // GetTupleElement does not create buffers. return Status::OK(); } @@ -99,24 +98,19 @@ Status LogicalBufferAnalysis::HandleCopy(HloInstruction* copy) { return Status::OK(); } -Status LogicalBufferAnalysis::HandleBitcast(HloInstruction* bitcast) { +Status LogicalBufferAnalysis::HandleBitcast(HloInstruction*) { // A kBitcast instruction aliases its operand. That is, the buffer of its // result *is* the buffer of its operand. return Status::OK(); } -Status LogicalBufferAnalysis::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status LogicalBufferAnalysis::HandleTuple(HloInstruction* tuple) { // A Tuple instruction only creates the top-level buffer. NewLogicalBuffer(tuple, /*index=*/{}); return Status::OK(); } -Status LogicalBufferAnalysis::HandleSelect(HloInstruction* select, - HloInstruction* /*pred*/, - HloInstruction* on_true, - HloInstruction* on_false) { +Status LogicalBufferAnalysis::HandleSelect(HloInstruction* select) { // Select allocates a new buffer and then shallow copies the on_true or // on_false buffer into this new buffer. NewLogicalBuffer(select, /*index=*/{}); diff --git a/tensorflow/compiler/xla/service/logical_buffer_analysis.h b/tensorflow/compiler/xla/service/logical_buffer_analysis.h index de9fe1b0a4ed3f6f8c466050520a9c4889793c62..a82e83ec5c3d2b0e011d85f3d03bea8fca870154 100644 --- a/tensorflow/compiler/xla/service/logical_buffer_analysis.h +++ b/tensorflow/compiler/xla/service/logical_buffer_analysis.h @@ -56,16 +56,11 @@ class LogicalBufferAnalysis : public DfsHloVisitorWithDefault { void NewLogicalBuffer(HloInstruction* instruction, const ShapeIndex& index); Status DefaultAction(HloInstruction* hlo_instruction) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleSelect(HloInstruction* select) override; // A map from the buffer ID to the logical buffer std::vector> logical_buffers_; diff --git a/tensorflow/compiler/xla/service/platform_util.cc b/tensorflow/compiler/xla/service/platform_util.cc index 4f915a0c2eeaca0fe077a907571c8379992185eb..3a1818de82d3fd305e2c6b3bd1f2cf8125806a75 100644 --- a/tensorflow/compiler/xla/service/platform_util.cc +++ b/tensorflow/compiler/xla/service/platform_util.cc @@ -84,15 +84,6 @@ PlatformUtil::GetSupportedPlatforms() { return NotFound("no platforms found"); } else if (platforms.size() == 1) { return platforms[0]; - } else if (platforms.size() == 2) { - // In the service we always link the cpu backend for ComputeConstant. So if - // one of the two platforms is CPU then pick the other (non-cpu) platform as - // the default. - if (platforms[0]->id() == se::host::kHostPlatformId) { - return platforms[1]; - } else if (platforms[1]->id() == se::host::kHostPlatformId) { - return platforms[0]; - } } // Multiple platforms present and we can't pick a reasonable default. diff --git a/tensorflow/compiler/xla/service/platform_util.h b/tensorflow/compiler/xla/service/platform_util.h index fe0281a69a441b5462470e88bd3ad73784a8da35..eac573703085aca2801885cd9abbe0022f1c029e 100644 --- a/tensorflow/compiler/xla/service/platform_util.h +++ b/tensorflow/compiler/xla/service/platform_util.h @@ -36,12 +36,7 @@ class PlatformUtil { // Convenience function which returns the default supported platform. If // exactly one supported platform is present, then this platform is the - // default platform. If exactly two supported platforms are present and one - // platform is CPU (host) then the non-CPU platform is default. This logic is - // used because the XLA service always links in the CPU backend to run - // ComputeConstant, so if exactly one other platform is linked in, we assume - // the intent is to execute on that non-CPU platform. If none of these - // conditions are met the function returns an error. + // default platform. Otherwise returns an error. static StatusOr GetDefaultPlatform(); // Returns a vector of StreamExecutors for the given platform. The vector is diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 0fbc2f2fec64917f5117dc5021c5e0a5b0f4367e..bac33d8102e07766531a4ce6eac77aff4971bfef 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -1415,9 +1415,9 @@ tensorflow::Status Service::Op(const OpRequest* arg, OpResponse* result) { // proto in the above switch statement. TF_ASSIGN_OR_RETURN(ComputationDataHandle handle, handle_status); TF_RETURN_IF_ERROR(computation->SetOpMetadata(handle, arg->metadata())); - TF_RETURN_IF_ERROR( - computation->SetOpDeviceAssignment(handle, arg->device_assignment())); - + if (arg->has_sharding()) { + TF_RETURN_IF_ERROR(computation->SetOpSharding(handle, arg->sharding())); + } return tensorflow::Status::OK(); } diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc index f3c8e3aff38dc389b05bef8fa69410ee06564b34..791d17365b1d756714b5feb0439e6919d9f23edc 100644 --- a/tensorflow/compiler/xla/service/shape_inference.cc +++ b/tensorflow/compiler/xla/service/shape_inference.cc @@ -53,6 +53,8 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) { return UNOP_EXP; case HloOpcode::kFloor: return UNOP_FLOOR; + case HloOpcode::kImag: + return UNOP_IMAG; case HloOpcode::kIsFinite: return UNOP_IS_FINITE; case HloOpcode::kLog: @@ -61,6 +63,8 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) { return UNOP_NOT; case HloOpcode::kNegate: return UNOP_NEGATE; + case HloOpcode::kReal: + return UNOP_REAL; case HloOpcode::kRoundNearestAfz: return UNOP_ROUND_NEAREST_AFZ; case HloOpcode::kSign: @@ -81,6 +85,10 @@ UnaryOperation OpcodeToUnaryOperation(HloOpcode opcode) { // opcode. BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) { switch (opcode) { + case HloOpcode::kAtan2: + return BINOP_ATAN2; + case HloOpcode::kComplex: + return BINOP_COMPLEX; case HloOpcode::kDot: return BINOP_DOT; case HloOpcode::kMultiply: @@ -89,8 +97,6 @@ BinaryOperation OpcodeToBinaryOperation(HloOpcode opcode) { return BINOP_ADD; case HloOpcode::kSubtract: return BINOP_SUB; - case HloOpcode::kIndex: - return BINOP_INDEX; case HloOpcode::kDivide: return BINOP_DIV; case HloOpcode::kEq: @@ -136,8 +142,6 @@ TernaryOperation OpcodeToTernaryOperation(HloOpcode opcode) { return TRIOP_CLAMP; case HloOpcode::kSelect: return TRIOP_SELECT; - case HloOpcode::kUpdate: - return TRIOP_UPDATE; default: LOG(FATAL) << "unhandled opcode " << opcode; } @@ -309,19 +313,41 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, switch (operation) { case UNOP_FLOOR: case UNOP_CEIL: + if (!ShapeUtil::ElementIsFloating(arg)) { + return InvalidArgument( + "expected element type in shape to be floating for floor/ceil " + "operation; got %s", + PrimitiveType_Name(arg.element_type()).c_str()); + } + return arg; case UNOP_COS: case UNOP_SIN: case UNOP_EXP: case UNOP_LOG: case UNOP_TANH: - if (!ShapeUtil::ElementIsFloating(arg)) { + if (!ShapeUtil::ElementIsFloating(arg) && + !ShapeUtil::ElementIsComplex(arg)) { return InvalidArgument( - "expected element type in shape to be floating for exp/log/tanh " - "operation; got %s", + "expected element type in shape to be floating or complex for " + "sin/cos/exp/log/tanh operation; got %s", PrimitiveType_Name(arg.element_type()).c_str()); } return arg; + case UNOP_REAL: + case UNOP_IMAG: + if (!ShapeUtil::ElementIsComplex(arg)) { + return InvalidArgument( + "expected element type in shape to be complex for real/imag " + "operation; got %s", + PrimitiveType_Name(arg.element_type()).c_str()); + } + return ShapeUtil::ChangeElementType(arg, F32); case UNOP_ABS: + if (ShapeUtil::ElementIsComplex(arg)) { + return ShapeUtil::ChangeElementType( + arg, primitive_util::ComplexComponentType(arg.element_type())); + } + return arg; case UNOP_NEGATE: case UNOP_ROUND_NEAREST_AFZ: case UNOP_SIGN: @@ -464,7 +490,10 @@ StatusOr InferWindowOutputShape(const Shape& base_shape, } if (ShapeUtil::Rank(operand_shape) != padding_config.dimensions_size()) { return InvalidArgument( - "the rank of the operand and the padding configuration do not match."); + "The rank of the operand and the padding configuration do not match: " + "%s vs %s", + ShapeUtil::HumanString(operand_shape).c_str(), + padding_config.ShortDebugString().c_str()); } if (operand_shape.element_type() != padding_value_shape.element_type()) { return InvalidArgument( @@ -750,6 +779,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( case BINOP_MIN: case BINOP_SUB: case BINOP_ADD: + case BINOP_ATAN2: case BINOP_POW: case BINOP_DIV: case BINOP_REM: @@ -760,6 +790,22 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( return InferElementwiseBinaryOpShape(operation, lhs, rhs, broadcast_dimensions); + case BINOP_COMPLEX: { + if (!ShapeUtil::ElementIsFloating(lhs)) { + return InvalidArgument( + "expected element type in shape to be floating for complex compose " + "operation; got %s", + PrimitiveType_Name(lhs.element_type()).c_str()); + } + TF_ASSIGN_OR_RETURN(const Shape& shape, + InferElementwiseBinaryOpShape(operation, lhs, rhs, + broadcast_dimensions)); + if (lhs.element_type() == F32) { + return ShapeUtil::ChangeElementType(shape, C64); + } else { + return Unimplemented("complex component type not supported"); + } + } case BINOP_AND: case BINOP_OR: if (lhs.element_type() != PRED && @@ -782,17 +828,6 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( broadcast_dimensions)); return ShapeUtil::ChangeElementType(shape, PRED); } - case BINOP_INDEX: - if (ShapeUtil::Rank(lhs) > 0 && ShapeUtil::Rank(rhs) == 0) { - tensorflow::gtl::ArraySlice dimensions = - AsInt64Slice(lhs.dimensions()); - dimensions.pop_front(); - return ShapeUtil::MakeShape(lhs.element_type(), dimensions); - } - return Unimplemented("cannot infer shape for operation: %s <%s> %s", - ShapeUtil::HumanString(lhs).c_str(), - BinaryOperation_Name(operation).c_str(), - ShapeUtil::HumanString(rhs).c_str()); default: return Unimplemented( "not yet implemented; infer binary op shape: %s; lhs: %s; rhs: %s", @@ -819,14 +854,6 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( return InferClampShape(lhs, rhs, ehs); case TRIOP_SELECT: return InferSelectShape(lhs, rhs, ehs); - case TRIOP_UPDATE: - TF_RETURN_IF_ERROR( - ExpectNotTupleOrOpaque(lhs, "lhs of ternary operation")); - TF_RETURN_IF_ERROR( - ExpectNotTupleOrOpaque(rhs, "rhs of ternary operation")); - TF_RETURN_IF_ERROR( - ExpectNotTupleOrOpaque(ehs, "ehs of ternary operation")); - return lhs; default: return InvalidArgument("unknown operation %s", TernaryOperation_Name(operation).c_str()); @@ -1382,14 +1409,8 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( "Window: %s", window.DebugString().c_str()); } - int num_spatial_dims = dnums.spatial_dimensions_size(); - if (num_spatial_dims < 1) { - return InvalidArgument( - "Convolution requires at least one spatial dimension.\n" - "Window: %s", - window.DebugString().c_str()); - } + const int num_spatial_dims = dnums.spatial_dimensions_size(); if (window.dimensions_size() != num_spatial_dims) { return InvalidArgument( "Window must have same number of dimensions as dimension numbers.\n" @@ -1397,7 +1418,7 @@ ShapeInference::InferDegenerateDimensionBroadcastShape( window.DebugString().c_str(), dnums.DebugString().c_str()); } - int num_dims = num_spatial_dims + 2; + const int num_dims = num_spatial_dims + 2; if (ShapeUtil::Rank(lhs) != num_dims) { return InvalidArgument( "The LHS argument to a convolution should have rank %d.\n" diff --git a/tensorflow/compiler/xla/service/shape_inference_test.cc b/tensorflow/compiler/xla/service/shape_inference_test.cc index 8df4a73229df25043d5490b0336b65955d4f4eed..d12f7bd1453890db3280e54719a6ce811006336d 100644 --- a/tensorflow/compiler/xla/service/shape_inference_test.cc +++ b/tensorflow/compiler/xla/service/shape_inference_test.cc @@ -35,6 +35,7 @@ class ShapeInferenceTest : public ::testing::Test { // Some handy scalar shapes. const Shape s32_ = ShapeUtil::MakeShape(S32, {}); const Shape f32_ = ShapeUtil::MakeShape(F32, {}); + const Shape f64_ = ShapeUtil::MakeShape(F64, {}); const Shape pred_ = ShapeUtil::MakeShape(PRED, {}); // Some handy vector and matrix shapes of F32 type. @@ -251,6 +252,44 @@ TEST_F(ShapeInferenceTest, ClampBadShapes) { .ok()); } +TEST_F(ShapeInferenceTest, Complex) { + auto complex_shape = [&](const Shape& lhs, const Shape& rhs, + const tensorflow::gtl::ArraySlice& bcast) { + return ShapeInference::InferBinaryOpShape(BinaryOperation::BINOP_COMPLEX, + lhs, rhs, bcast); + }; + // Inputs must be FP. + ASSERT_FALSE(complex_shape(s32_, s32_, {}).ok()); + ASSERT_FALSE(complex_shape(pred_, pred_, {}).ok()); + // Component types must match. + ASSERT_FALSE(complex_shape(f32_, f64_, {}).ok()); + // Only F32->C64 supported. + ASSERT_FALSE(complex_shape(f64_, f64_, {}).ok()); + // Validate correct uses. + Shape c64_32 = ShapeUtil::MakeShape(C64, {32}); + TF_ASSERT_OK_AND_ASSIGN(Shape result, complex_shape(f32_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, ShapeUtil::MakeShape(C64, {}))); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(vector_32_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32)); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(f32_, vector_32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32)); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(vector_32_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32)); + + Shape c64_32_64 = ShapeUtil::MakeShape(C64, {32, 64}); + TF_ASSERT_OK_AND_ASSIGN(result, + complex_shape(vector_64_, matrix_32_64_, {1})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); + TF_ASSERT_OK_AND_ASSIGN(result, + complex_shape(matrix_32_64_, vector_64_, {1})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); + TF_ASSERT_OK_AND_ASSIGN(result, + complex_shape(matrix_32_64_, matrix_32_64_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); + TF_ASSERT_OK_AND_ASSIGN(result, complex_shape(matrix_32_64_, f32_, {})); + ASSERT_TRUE(ShapeUtil::Equal(result, c64_32_64)); +} + TEST_F(ShapeInferenceTest, VariadicOpTuplify) { StatusOr result = ShapeInference::InferVariadicOpShape( VariadicOperation::VAROP_TUPLE, {&s32_, &f32_}); diff --git a/tensorflow/compiler/xla/service/transpose_folding.cc b/tensorflow/compiler/xla/service/transpose_folding.cc index 816c8a7485bb9c5c12d3dc9e17404c74460113f5..8c2640adf52f10c387e7a9c09c0d73a09c054919 100644 --- a/tensorflow/compiler/xla/service/transpose_folding.cc +++ b/tensorflow/compiler/xla/service/transpose_folding.cc @@ -58,14 +58,32 @@ TransposeFolding::OperandIndices CanFoldOperandsIntoConvolution( return {}; } - // We only support folding the RHS. - const int64 kRhsOperandIndex = 1; - auto& operand = *convolution.operand(kRhsOperandIndex); - if (operand.opcode() == HloOpcode::kTranspose && operand.user_count() == 1) { - return transposable_conv_operands(convolution, {kRhsOperandIndex}); + const ConvolutionDimensionNumbers& dnums = + convolution.convolution_dimension_numbers(); + + TransposeFolding::OperandIndices operand_set; + for (int64 i = 0; i < convolution.operand_count(); ++i) { + auto& operand = *convolution.operand(i); + if (operand.opcode() == HloOpcode::kTranspose && + operand.user_count() == 1) { + const auto& transpose_dimensions = operand.dimensions(); + // We can transpose the LHS so long as it doesn't move around spatial + // dimensions because ConvolutionDimensionNumbers doesn't have different + // fields for input and output spatial dimensions. + if (i == 0 && + std::any_of(dnums.spatial_dimensions().begin(), + dnums.spatial_dimensions().end(), + [&](const int64 spatial_dimension) { + return transpose_dimensions[spatial_dimension] != + spatial_dimension; + })) { + continue; + } + operand_set.push_back(i); + } } - return {}; + return transposable_conv_operands(convolution, operand_set); } using InstructionOperandsPair = @@ -98,40 +116,61 @@ bool FoldTransposeIntoDot(InstructionOperandsPair pair) { // Returns whether the module is changed. bool FoldTransposeIntoConvolution(InstructionOperandsPair pair) { auto& convolution = *pair.first; - - // We only support fusing the RHS transpose into convolution. - // - // ConvolutionDimensionNumbers doesn't make enough of a distinction between - // the output and the activations. - // - // TODO(b/37125184): Support transposing the LHS too. - if (pair.second.size() != 1 || pair.second.front() != 1) { - return false; - } + auto& operand_indices = pair.second; const ConvolutionDimensionNumbers& dnums = convolution.convolution_dimension_numbers(); - HloInstruction& transpose = *convolution.mutable_operand(1); - CHECK_EQ(transpose.opcode(), HloOpcode::kTranspose); - const auto& transpose_dimensions = transpose.dimensions(); - HloInstruction& transpose_operand = *transpose.mutable_operand(0); - - // Everything remains the same except for the kernel dimension numbers. We - // need to apply the transpose permutation to the original shape to figure out - // what the new logical dimensions are. ConvolutionDimensionNumbers new_dnums = dnums; - new_dnums.set_kernel_input_feature_dimension( - transpose_dimensions[dnums.kernel_input_feature_dimension()]); - new_dnums.set_kernel_output_feature_dimension( - transpose_dimensions[dnums.kernel_output_feature_dimension()]); - for (auto& kernel_spatial_dimension : - *new_dnums.mutable_kernel_spatial_dimensions()) { - kernel_spatial_dimension = transpose_dimensions[kernel_spatial_dimension]; + + HloInstruction* new_lhs; + const int64 kLhsIdx = 0; + if (std::find(operand_indices.begin(), operand_indices.end(), kLhsIdx) != + operand_indices.end()) { + HloInstruction& transpose = *convolution.mutable_operand(kLhsIdx); + const auto& transpose_dimensions = transpose.dimensions(); + HloInstruction& transpose_operand = *transpose.mutable_operand(0); + + // Everything remains the same except for the input/output dimension + // numbers. We need to apply the transpose permutation to the original shape + // to figure out what the new logical dimensions are. + new_dnums.set_input_batch_dimension( + transpose_dimensions[dnums.input_batch_dimension()]); + new_dnums.set_input_feature_dimension( + transpose_dimensions[dnums.input_feature_dimension()]); + for (const auto& spatial_dimension : dnums.spatial_dimensions()) { + CHECK_EQ(spatial_dimension, transpose_dimensions[spatial_dimension]); + } + new_lhs = &transpose_operand; + } else { + new_lhs = convolution.mutable_operand(kLhsIdx); + } + + HloInstruction* new_rhs; + const int64 kRhsIdx = 1; + if (std::find(operand_indices.begin(), operand_indices.end(), kRhsIdx) != + operand_indices.end()) { + HloInstruction& transpose = *convolution.mutable_operand(kRhsIdx); + const auto& transpose_dimensions = transpose.dimensions(); + HloInstruction& transpose_operand = *transpose.mutable_operand(0); + + // Everything remains the same except for the kernel dimension numbers. We + // need to apply the transpose permutation to the original shape to figure + // out what the new logical dimensions are. + new_dnums.set_kernel_input_feature_dimension( + transpose_dimensions[dnums.kernel_input_feature_dimension()]); + new_dnums.set_kernel_output_feature_dimension( + transpose_dimensions[dnums.kernel_output_feature_dimension()]); + for (auto& kernel_spatial_dimension : + *new_dnums.mutable_kernel_spatial_dimensions()) { + kernel_spatial_dimension = transpose_dimensions[kernel_spatial_dimension]; + } + new_rhs = &transpose_operand; + } else { + new_rhs = convolution.mutable_operand(kRhsIdx); } auto new_conv = HloInstruction::CreateConvolve( - convolution.shape(), convolution.mutable_operand(0), &transpose_operand, - convolution.window(), new_dnums); + convolution.shape(), new_lhs, new_rhs, convolution.window(), new_dnums); TF_CHECK_OK(convolution.parent()->ReplaceWithNewInstruction( &convolution, std::move(new_conv))); diff --git a/tensorflow/compiler/xla/service/transpose_folding_test.cc b/tensorflow/compiler/xla/service/transpose_folding_test.cc index a6161b46460068b83fa3f0762e49a10a83b1471c..00462f9be1e9beb2f2694060ebfaa70b0b9dd4a0 100644 --- a/tensorflow/compiler/xla/service/transpose_folding_test.cc +++ b/tensorflow/compiler/xla/service/transpose_folding_test.cc @@ -313,8 +313,7 @@ TEST_F(TransposeFoldingTest, FoldConvComplexTransposeRhs) { new_conv->convolution_dimension_numbers().kernel_spatial_dimensions(1)); } -// Test that a transpose of the activations does not get folded into -// convolution. +// Test that a transpose of the activations gets folded into convolution. TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) { auto builder = HloComputation::Builder("entry_computation"); HloInstruction* x = builder.AddInstruction(HloInstruction::CreateParameter( @@ -348,18 +347,25 @@ TEST_F(TransposeFoldingTest, FoldConvTransposeLhs) { module.AddEntryComputation(builder.Build(conv)); FoldTranspose(&module); - // Instructions after folding: transpose_x, y, and the convolution. + // Instructions after folding: x, y, and the convolution. std::unordered_set instruction_set( entry_computation->instructions().begin(), entry_computation->instructions().end()); - CHECK_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation."; - CHECK_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation."; - CHECK_EQ(1, instruction_set.erase(transpose_x)) - << "transpose_x is not in entry_computation."; - CHECK_EQ(1, instruction_set.erase(conv)) - << "transpose_x is not in entry_computation."; - CHECK_EQ(0, instruction_set.size()) - << "entry_computation should contain exactly 4 instructions."; + EXPECT_EQ(1, instruction_set.erase(x)) << "x is not in entry_computation."; + EXPECT_EQ(1, instruction_set.erase(y)) << "y is not in entry_computation."; + EXPECT_EQ(1, instruction_set.size()) + << "entry_computation should contain exactly 3 instructions."; + HloInstruction* new_conv = *instruction_set.begin(); + EXPECT_EQ(HloOpcode::kConvolution, new_conv->opcode()); + EXPECT_EQ(dnums.input_feature_dimension(), + new_conv->convolution_dimension_numbers().input_batch_dimension()); + EXPECT_EQ( + dnums.input_batch_dimension(), + new_conv->convolution_dimension_numbers().input_feature_dimension()); + EXPECT_EQ(dnums.spatial_dimensions(0), + new_conv->convolution_dimension_numbers().spatial_dimensions(0)); + EXPECT_EQ(dnums.spatial_dimensions(1), + new_conv->convolution_dimension_numbers().spatial_dimensions(1)); } } // namespace diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc index f7dee93aad86bd631a546318faad7f6756d563c7..df537bd7c15a1f15ed77ca9be6ce70fbfd2e63be 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.cc @@ -200,13 +200,14 @@ Status TuplePointsToAnalysis::DefaultAction(HloInstruction* hlo_instruction) { } Status TuplePointsToAnalysis::HandleGetTupleElement( - HloInstruction* get_tuple_element, HloInstruction* operand) { + HloInstruction* get_tuple_element) { // GetTupleElement forwards a pointer to a particular element of the tuple // operand. int64 element_index = get_tuple_element->tuple_index(); PointsToSet& points_to_set = CreateEmptyPointsToSet(get_tuple_element); - const PointsToSet& operand_points_to_set = *PerInst(operand)->points_to_set; + const PointsToSet& operand_points_to_set = + *PerInst(get_tuple_element->operand(0))->points_to_set; // Copy the points-to set (and tuple sources) at index {element_index} of the // operand to the points-to set for this GetTupleElement instruction. @@ -252,9 +253,8 @@ Status TuplePointsToAnalysis::HandleBitcast(HloInstruction* bitcast) { return Status::OK(); } -Status TuplePointsToAnalysis::HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) { +Status TuplePointsToAnalysis::HandleTuple(HloInstruction* tuple) { + tensorflow::gtl::ArraySlice operands(tuple->operands()); PointsToSet& points_to_set = CreateEmptyPointsToSet(tuple); points_to_set.AddPointedToBuffer( logical_buffer_analysis_->GetBuffer(tuple, /*index=*/{}), @@ -292,10 +292,7 @@ Status TuplePointsToAnalysis::HandleTuple( return Status::OK(); } -Status TuplePointsToAnalysis::HandleSelect(HloInstruction* select, - HloInstruction* /*pred*/, - HloInstruction* on_true, - HloInstruction* on_false) { +Status TuplePointsToAnalysis::HandleSelect(HloInstruction* select) { // Select allocates a new buffer and then shallow copies the on_true or // on_false buffer into this new buffer. Which side is chosen cannot be // determined statically so conservatively set the points-to set to the union @@ -303,6 +300,8 @@ Status TuplePointsToAnalysis::HandleSelect(HloInstruction* select, // // First create a copy of the on_true points-to set (and tuple sources), then // add in elements of the on_false points-to set (tuple sources). + auto on_true = select->operand(1); + auto on_false = select->operand(2); PointsToSet& points_to_set = CreateCopiedPointsToSet(select, on_true); const PointsToSet& false_points_to_set = *PerInst(on_false)->points_to_set; points_to_set.ForEachMutableElement( diff --git a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h index be457329521c62dc86d60b09cf189c43e6f1dde1..e6157a1ed11b5df24458fe820a4e0e329eb86ae4 100644 --- a/tensorflow/compiler/xla/service/tuple_points_to_analysis.h +++ b/tensorflow/compiler/xla/service/tuple_points_to_analysis.h @@ -44,7 +44,7 @@ namespace xla { // A class describing the source(s) of the Buffer(s) contained in the output of // a particular HLO instruction. The structure of PointsToSet mirrors the -// structure of the instruction's shape which may be an arbitrary tree (eg, a +// structure of the instruction's shape, which may be an arbitrary tree (eg, a // nested tuple). Each node in this tree corresponds to a single buffer in the // instruction's output and contains the set of Buffers which might define // the corresponding buffer. @@ -148,7 +148,7 @@ class PointsToSet { ShapeTree tree_; // PointsToSet contains references (const LogicalBuffer*) to elements within - // TuplePointsToAnalysis so disable copying. + // TuplePointsToAnalysis, so disable copying. TF_DISALLOW_COPY_AND_ASSIGN(PointsToSet); }; @@ -247,16 +247,11 @@ class TuplePointsToAnalysis : public DfsHloVisitorWithDefault { Status VerifyBuffer(const LogicalBuffer& buffer) const; Status DefaultAction(HloInstruction* hlo_instruction) override; - Status HandleTuple( - HloInstruction* tuple, - tensorflow::gtl::ArraySlice operands) override; - Status HandleGetTupleElement(HloInstruction* get_tuple_element, - HloInstruction* operand) override; + Status HandleTuple(HloInstruction* tuple) override; + Status HandleGetTupleElement(HloInstruction* get_tuple_element) override; Status HandleBitcast(HloInstruction* bitcast) override; Status HandleCopy(HloInstruction* copy) override; - Status HandleSelect(HloInstruction* select, HloInstruction* pred, - HloInstruction* on_true, - HloInstruction* on_false) override; + Status HandleSelect(HloInstruction* select) override; string ToString() const; diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc index b3506b72bf5ab1aa27704c18c8a1dc69881caf71..006c814996df9b209e6cd4d75bc04689c4e297c5 100644 --- a/tensorflow/compiler/xla/service/user_computation.cc +++ b/tensorflow/compiler/xla/service/user_computation.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include #include +#include #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal_util.h" @@ -54,6 +55,8 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) { return HloOpcode::kExp; case UNOP_FLOOR: return HloOpcode::kFloor; + case UNOP_IMAG: + return HloOpcode::kImag; case UNOP_IS_FINITE: return HloOpcode::kIsFinite; case UNOP_LOG: @@ -62,6 +65,8 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) { return HloOpcode::kNot; case UNOP_NEGATE: return HloOpcode::kNegate; + case UNOP_REAL: + return HloOpcode::kReal; case UNOP_ROUND_NEAREST_AFZ: return HloOpcode::kRoundNearestAfz; case UNOP_SIGN: @@ -79,6 +84,10 @@ HloOpcode UnaryOperationToHloOpcode(UnaryOperation unop) { HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) { switch (binop) { + case BINOP_ATAN2: + return HloOpcode::kAtan2; + case BINOP_COMPLEX: + return HloOpcode::kComplex; case BINOP_DOT: return HloOpcode::kDot; case BINOP_MUL: @@ -87,8 +96,6 @@ HloOpcode BinaryOperationToHloOpcode(BinaryOperation binop) { return HloOpcode::kAdd; case BINOP_SUB: return HloOpcode::kSubtract; - case BINOP_INDEX: - return HloOpcode::kIndex; case BINOP_DIV: return HloOpcode::kDivide; case BINOP_EQ: @@ -132,8 +139,6 @@ HloOpcode TernaryOperationToHloOpcode(TernaryOperation triop) { return HloOpcode::kClamp; case TRIOP_SELECT: return HloOpcode::kSelect; - case TRIOP_UPDATE: - return HloOpcode::kUpdate; default: LOG(FATAL) << "unhandled operation " << triop; } @@ -1308,20 +1313,19 @@ Status UserComputation::SetOpMetadata(const ComputationDataHandle& handle, return Status::OK(); } -Status UserComputation::SetOpDeviceAssignment( - const ComputationDataHandle& handle, - const OpDeviceAssignment& device_assignment) { +Status UserComputation::SetOpSharding(const ComputationDataHandle& handle, + const OpSharding& sharding) { tensorflow::mutex_lock lock(mutex_); int64 handle_value = handle.handle(); if (session_computation_.requests().count(handle_value) == 0) { - return InvalidArgument("Invalid handle in SetOpDeviceAssignment (%lld)", + return InvalidArgument("Invalid handle in SetOpSharding (%lld)", handle_value); } *session_computation_.mutable_requests() ->at(handle_value) .mutable_request() - ->mutable_device_assignment() = device_assignment; + ->mutable_sharding() = sharding; return Status::OK(); } @@ -1843,10 +1847,17 @@ UserComputation::GetEmbeddedComputations( XLA_VLOG_LINES(3, session_computation_.DebugString()); std::vector computations; + std::vector sorted_handles; for (const auto& handle_request : session_computation_.requests()) { - int64 handle_value = handle_request.first; + sorted_handles.push_back(handle_request.first); + } + std::sort(sorted_handles.begin(), sorted_handles.end()); + for (int64 handle : sorted_handles) { + const auto& handle_request = session_computation_.requests().find(handle); + CHECK(handle_request != session_computation_.requests().end()); + int64 handle_value = handle_request->first; if (handle_value <= version) { - const OperationRequest& request = handle_request.second; + const OperationRequest& request = handle_request->second; switch (request.request().op_case()) { case OpRequest::kCallRequest: { CHECK_EQ(1, request.embedded_computation_versions_size()); @@ -2504,7 +2515,9 @@ HloInstruction* ComputationLowerer::ImplicitBroadcastToExplicitBroadcast( if (ShapeUtil::IsScalar(operand->shape())) { HloInstruction* broadcast = hlo_builder_.AddInstruction( HloInstruction::CreateBroadcast(broadcast_shape, operand, {})); - broadcast->set_device_assignment(operand->device_assignment()); + if (operand->has_sharding()) { + broadcast->set_sharding(operand->sharding()); + } return broadcast; } // Do explicit broadcast for degenerate broadcast. @@ -2522,12 +2535,16 @@ HloInstruction* ComputationLowerer::ImplicitBroadcastToExplicitBroadcast( ShapeUtil::MakeShape(operand->shape().element_type(), reshaped_dimensions), operand)); - reshaped_operand->set_device_assignment(operand->device_assignment()); + if (operand->has_sharding()) { + reshaped_operand->set_sharding(operand->sharding()); + } // Broadcast 'reshape' up to the larger size. HloInstruction* broadcast = hlo_builder_.AddInstruction(HloInstruction::CreateBroadcast( broadcast_shape, reshaped_operand, broadcast_dimensions)); - broadcast->set_device_assignment(operand->device_assignment()); + if (operand->has_sharding()) { + broadcast->set_sharding(operand->sharding()); + } return broadcast; } @@ -2542,8 +2559,11 @@ void ComputationLowerer::Visit( HloInstruction* hlo_instruction = hlo_builder_.AddInstruction(std::move(instruction)); hlo_instruction->set_metadata(request.request().metadata()); - hlo_instruction->set_device_assignment( - request.request().device_assignment()); + if (request.request().has_sharding()) { + OpSharding op_sharding = request.request().sharding(); + hlo_instruction->set_sharding( + HloSharding::FromProto(op_sharding).ValueOrDie()); + } return hlo_instruction; }; auto lookup_instruction = [&](const ComputationDataHandle& handle) { diff --git a/tensorflow/compiler/xla/service/user_computation.h b/tensorflow/compiler/xla/service/user_computation.h index 6f3bf430fc948732bd771ac3efb60ac9791076d2..dabf68e298ed2600d5248b7b8c7b1e014efedb14 100644 --- a/tensorflow/compiler/xla/service/user_computation.h +++ b/tensorflow/compiler/xla/service/user_computation.h @@ -262,8 +262,8 @@ class UserComputation { const OpMetadata& metadata); // Sets the device assignment on the Hlo instruction referenced by 'handle'. - Status SetOpDeviceAssignment(const ComputationDataHandle& handle, - const OpDeviceAssignment& device_assignment); + Status SetOpSharding(const ComputationDataHandle& handle, + const OpSharding& sharding); // Builds a HLO computation from the UserComputation. The parameter "resolver" // is a function which returns a pointer to the HloComputation corresponding diff --git a/tensorflow/compiler/xla/service/user_computation_test.cc b/tensorflow/compiler/xla/service/user_computation_test.cc index 43a857935a4548482eba90c01422525e911787a6..5afaf226ae0cce7e9afc966c6b4adf838aeebc91 100644 --- a/tensorflow/compiler/xla/service/user_computation_test.cc +++ b/tensorflow/compiler/xla/service/user_computation_test.cc @@ -224,10 +224,13 @@ TEST_F(UserComputationTest, CheckImplicitBroadcastToExplicitBroadcast) { TF_ASSERT_OK_AND_ASSIGN(ComputationDataHandle b_handle, computation.AddParameterInstruction(b_request)); - OpDeviceAssignment assignment; - assignment.set_has_device(true); - assignment.set_device(7); - TF_EXPECT_OK(computation.SetOpDeviceAssignment(b_handle, assignment)); + const int64 kDevice = 7; + OpSharding sharding; + sharding.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + sharding.add_tile_assignment_dimensions(1); + sharding.add_tile_assignment_devices(kDevice); + + TF_EXPECT_OK(computation.SetOpSharding(b_handle, sharding)); BinaryOpRequest add; add.set_binop(BINOP_ADD); @@ -260,12 +263,10 @@ TEST_F(UserComputationTest, CheckImplicitBroadcastToExplicitBroadcast) { const HloInstruction* broadcast = hlo_computation->root_instruction()->operand(1); - EXPECT_TRUE(broadcast->device_assignment().has_device()); - EXPECT_EQ(assignment.device(), broadcast->device_assignment().device()); + EXPECT_TRUE(broadcast->has_sharding()); const HloInstruction* reshape = broadcast->operand(0); - EXPECT_TRUE(reshape->device_assignment().has_device()); - EXPECT_EQ(assignment.device(), reshape->device_assignment().device()); + EXPECT_TRUE(reshape->has_sharding()); } TEST_F(UserComputationTest, EliminateDegenerateBroadcastAfterIndimBroadcast) { diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index 8e16056b239a9e1d1776bfe91f6e36862e0feeec..b5eb81dfc6a4117909dcb18fdbe61443b1a1eb95 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -102,6 +102,32 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { return true; } +// Constructs and returns the new shape with the given minor_to_major order in +// its Layout. +StatusOr MakeShapeWithLayoutInternal( + PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, + tensorflow::gtl::ArraySlice minor_to_major) { + if (dimensions.size() != minor_to_major.size()) { + return InvalidArgument("Dimensions size is %ld, but layout size is %ld.", + dimensions.size(), minor_to_major.size()); + } + if (element_type == OPAQUE || element_type == TUPLE) { + return InvalidArgument("Unsupported element type: %s", + PrimitiveType_Name(element_type).c_str()); + } + Shape shape = ShapeUtil::MakeShape(element_type, dimensions); + auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); + min2maj->Clear(); + for (int64 value : minor_to_major) { + min2maj->Add(value); + } + if (!shape.has_layout()) { + return InvalidArgument("Shape has no layout."); + } + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(shape)); + return shape; +} + } // namespace /* static */ bool ShapeUtil::Equal(const Shape& lhs, const Shape& rhs) { @@ -152,16 +178,8 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { /* static */ Shape ShapeUtil::MakeShapeWithLayout( PrimitiveType element_type, tensorflow::gtl::ArraySlice dimensions, tensorflow::gtl::ArraySlice minor_to_major) { - CHECK_EQ(dimensions.size(), minor_to_major.size()); - Shape shape = MakeShape(element_type, dimensions); - auto min2maj = shape.mutable_layout()->mutable_minor_to_major(); - min2maj->Clear(); - for (int64 value : minor_to_major) { - min2maj->Add(value); - } - DCHECK(shape.has_layout()); - TF_DCHECK_OK(ValidateShape(shape)); - return shape; + return MakeShapeWithLayoutInternal(element_type, dimensions, minor_to_major) + .ValueOrDie(); } /* static */ Shape ShapeUtil::MakeShapeWithMonotonicDim0MajorLayout( @@ -254,6 +272,7 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { case U16: case U32: case U64: + case C64: case TUPLE: case OPAQUE: return false; @@ -263,6 +282,10 @@ bool CompareShapes(const Shape& lhs, const Shape& rhs, bool compare_layouts) { } } +/* static */ bool ShapeUtil::ElementIsComplex(const Shape& shape) { + return primitive_util::IsComplexType(shape.element_type()); +} + /* static */ bool ShapeUtil::ElementIsFloating(const Shape& shape) { return primitive_util::IsFloatingPointType(shape.element_type()); } @@ -499,11 +522,10 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { // Extract the layout minor-to-major and set it. TF_ASSIGN_OR_RETURN(std::vector min2maj, comma_list_to_int64s(layout_string)); - TF_RET_CHECK(dimensions.size() == min2maj.size()); - result = - ShapeUtil::MakeShapeWithLayout(primitive_type, dimensions, min2maj); + TF_ASSIGN_OR_RETURN(result, MakeShapeWithLayoutInternal( + primitive_type, dimensions, min2maj)); } - TF_DCHECK_OK(ShapeUtil::ValidateShape(result)); + TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(result)); return std::move(result); } @@ -575,6 +597,8 @@ StatusOr ParseShapeStringInternal(tensorflow::StringPiece* s) { return sizeof(float); case F64: return sizeof(double); + case C64: + return sizeof(complex64); default: LOG(FATAL) << "Unhandled primitive type " << primitive_type; } diff --git a/tensorflow/compiler/xla/shape_util.h b/tensorflow/compiler/xla/shape_util.h index c5800acaf11a99be4545e2ad4330101e7971bd7c..8f8d4a73c9ecb3f4236f3877323ad1127bb0b9c2 100644 --- a/tensorflow/compiler/xla/shape_util.h +++ b/tensorflow/compiler/xla/shape_util.h @@ -291,6 +291,9 @@ class ShapeUtil { // Returns whether the element type of the shape is floating point. static bool ElementIsFloating(const Shape& shape); + // Returns whether the element type of the shape is complex. + static bool ElementIsComplex(const Shape& shape); + // Returns whether the element type has the given bit width. static bool ElementHasBitWidth(const Shape& shape, int bits); diff --git a/tensorflow/compiler/xla/shape_util_test.cc b/tensorflow/compiler/xla/shape_util_test.cc index 79945b9c77299b7006d014aed4507566e3c2c750..0ba542ad1bec290c35c52a8dd5177893770310fd 100644 --- a/tensorflow/compiler/xla/shape_util_test.cc +++ b/tensorflow/compiler/xla/shape_util_test.cc @@ -218,6 +218,10 @@ TEST(ShapeUtilTest, ByteSizeOfWithoutPadding) { EXPECT_EQ(8, ShapeUtil::ByteSizeOfPrimitiveType(F64)); EXPECT_EQ(8, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(F64, {}))); EXPECT_EQ(1600, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(F64, {10, 20}))); + + EXPECT_EQ(8, ShapeUtil::ByteSizeOfPrimitiveType(C64)); + EXPECT_EQ(8, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(C64, {}))); + EXPECT_EQ(1600, ShapeUtil::ByteSizeOf(ShapeUtil::MakeShape(C64, {10, 20}))); } TEST(ShapeUtilTest, ByteSizeOfWithPadding) { diff --git a/tensorflow/compiler/xla/test_helpers.h b/tensorflow/compiler/xla/test_helpers.h index 634cdb5aa29651b08090ff99f0a6cafb9facb645..17bae2e4f611268df824ce793c75ba1c95573455 100644 --- a/tensorflow/compiler/xla/test_helpers.h +++ b/tensorflow/compiler/xla/test_helpers.h @@ -62,9 +62,16 @@ inline const ::tensorflow::Status& GetStatus(const StatusOr& status) { #define EXPECT_IS_OK(expression) \ EXPECT_EQ(tensorflow::Status::OK(), \ xla::testing::internal_status::GetStatus(expression)) +#define EXPECT_IS_NOT_OK(expression) \ + EXPECT_NE(tensorflow::Status::OK(), \ + xla::testing::internal_status::GetStatus(expression)) #undef ASSERT_IS_OK #define ASSERT_IS_OK(expression) \ ASSERT_EQ(tensorflow::Status::OK(), \ xla::testing::internal_status::GetStatus(expression)) +#undef ASSERT_IS_NOT_OK +#define ASSERT_IS_NOT_OK(expression) \ + ASSERT_NE(tensorflow::Status::OK(), \ + xla::testing::internal_status::GetStatus(expression)) #endif // TENSORFLOW_COMPILER_XLA_TEST_HELPERS_H_ diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 769f509adc86011953e39185599ece1da0c84b22..4e1be24b61cc436b0baf62cc6e28ad8d13fe71ac 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -23,7 +23,6 @@ filegroup( ]), ) -load("//tensorflow/compiler/xla:xla.bzl", "export_dynamic_linkopts") load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test") load("//tensorflow/compiler/xla/tests:build_defs.bzl", "xla_test_library") load("//tensorflow/compiler/xla/tests:build_defs.bzl", "generate_backend_suites") @@ -102,28 +101,18 @@ cc_library( deps = [ ":literal_test_util", "//tensorflow/compiler/xla:shape_layout", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/legacy_flags:debug_options_flags", - "//tensorflow/compiler/xla/service", "//tensorflow/compiler/xla/service:backend", - "//tensorflow/compiler/xla/service:compiler", "//tensorflow/compiler/xla/service:computation_layout", - "//tensorflow/compiler/xla/service:computation_placer", - "//tensorflow/compiler/xla/service:executable", "//tensorflow/compiler/xla/service:hlo", - "//tensorflow/compiler/xla/service:hlo_execution_profile", - "//tensorflow/compiler/xla/service:hlo_graph_dumper", - "//tensorflow/compiler/xla/service:transfer_manager", - "//tensorflow/core:core_cpu_internal", + "//tensorflow/compiler/xla/service:hlo_runner", "//tensorflow/core:lib", "//tensorflow/core:stream_executor_no_cuda", "//tensorflow/core:test", - "//third_party/eigen3", ], ) @@ -931,6 +920,7 @@ xla_test( name = "reduce_window_test", timeout = "long", srcs = [], + tags = ["optonly"], xla_test_library_deps = [":reduce_window_test_library"], deps = [], ) @@ -998,13 +988,13 @@ xla_test( xla_test( name = "custom_call_test", srcs = ["custom_call_test.cc"], - linkopts = export_dynamic_linkopts, deps = [ "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:util", "//tensorflow/compiler/xla:xla_data_proto", "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service/cpu:custom_call_target_registry", "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", @@ -1411,8 +1401,10 @@ xla_test( "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:literal_test_util", "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:core_cpu_internal", "//tensorflow/core:lib", "//tensorflow/core:test", + "//third_party/eigen3", ], ) diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc index a60d3e50bd4dc78ed8715f8d7814668b95f3d38a..065bce7e3146c93568bbce2b0e7e23ddddc4ea31 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.cc +++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc @@ -254,7 +254,8 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus( tensorflow::gtl::ArraySlice arguments, const Shape* shape_with_layout) { TF_ASSIGN_OR_RETURN(auto computation, builder->Build()); - if (ShapeUtil::ElementIsFloating(expected.shape())) { + if (ShapeUtil::ElementIsFloating(expected.shape()) || + ShapeUtil::ElementIsComplex(expected.shape())) { LOG(WARNING) << "performing exact comparison of floating point numbers"; } else { TF_RET_CHECK(ShapeUtil::ElementIsIntegral(expected.shape()) || @@ -282,7 +283,8 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus( ComputationBuilder* builder, const Literal& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error, const Shape* shape_with_layout) { - TF_RET_CHECK(ShapeUtil::ElementIsFloating(expected.shape())); + TF_RET_CHECK(ShapeUtil::ElementIsFloating(expected.shape()) || + ShapeUtil::ElementIsComplex(expected.shape())); TF_ASSIGN_OR_RETURN(auto computation, builder->Build()); auto expect_near = [&](const Literal& actual, const string& error_message) { LiteralTestUtil::ExpectNear(expected, actual, error, error_message); diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h index 7fe1445b94097f762b777fc6936a0a1ab5a726c8..7cfc276ec19e3b177f87a08e716cb34b7676dd6b 100644 --- a/tensorflow/compiler/xla/tests/client_library_test_base.h +++ b/tensorflow/compiler/xla/tests/client_library_test_base.h @@ -361,8 +361,9 @@ void ClientLibraryTestBase::ComputeAndCompareR2( ComputationBuilder* builder, const Array2D& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error) { static_assert(std::is_same::value || - std::is_same::value, - "Floating point type required when specifying an ErrorSpec"); + std::is_same::value || + std::is_same::value, + "Float or complex type required when specifying an ErrorSpec"); std::unique_ptr expected_literal = Literal::CreateR2FromArray2D(expected); ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal, @@ -384,8 +385,9 @@ void ClientLibraryTestBase::ComputeAndCompareR3( ComputationBuilder* builder, const Array3D& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error) { static_assert(std::is_same::value || - std::is_same::value, - "Floating point type required when specifying an ErrorSpec"); + std::is_same::value || + std::is_same::value, + "Float or complex type required when specifying an ErrorSpec"); std::unique_ptr expected_literal = Literal::CreateR3FromArray3D(expected); ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal, @@ -407,8 +409,9 @@ void ClientLibraryTestBase::ComputeAndCompareR4( ComputationBuilder* builder, const Array4D& expected, tensorflow::gtl::ArraySlice arguments, ErrorSpec error) { static_assert(std::is_same::value || - std::is_same::value, - "Floating point type required when specifying an ErrorSpec"); + std::is_same::value || + std::is_same::value, + "Float or complex type required when specifying an ErrorSpec"); std::unique_ptr expected_literal = Literal::CreateR4FromArray4D(expected); ClientLibraryTestBase::ComputeAndCompareLiteral(builder, *expected_literal, diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc index a7089c2897bee2a10b698df910b4805456257949..0cc2e5fb7e655884f3334426a684dd3ce00d4052 100644 --- a/tensorflow/compiler/xla/tests/convolution_test.cc +++ b/tensorflow/compiler/xla/tests/convolution_test.cc @@ -508,21 +508,35 @@ XLA_TEST_F(ConvolutionTest, Convolve2D_1x3x3x5_3x3x5x5_Valid) { error_spec_); } -XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) { +struct Convolve1DTestParam { + int64 input_feature; + int64 output_feature; + int64 batch; + int64 window_size; + int64 num_windows; +}; + +class Convolve1D1WindowTest + : public ConvolutionTest, + public ::testing::WithParamInterface {}; + +XLA_TEST_P(Convolve1D1WindowTest, Convolve1D1Window) { ComputationBuilder builder(client_, TestName()); - int64 output_feature = 1; - int64 input_feature = 64; - int64 batch = 1; - int64 length = 1; - std::vector input_dims = {batch, 4 + length - 1, input_feature}; - std::vector filter_dims = {4, input_feature, output_feature}; + int64 input_feature = GetParam().input_feature; + int64 output_feature = GetParam().output_feature; + int64 batch = GetParam().batch; + int64 num_windows = GetParam().num_windows; + int64 window_size = GetParam().window_size; + std::vector input_dims = {batch, window_size + num_windows - 1, + input_feature}; + std::vector filter_dims = {window_size, input_feature, output_feature}; Shape input_shape = ShapeUtil::MakeShape(F32, input_dims); Shape filter_shape = ShapeUtil::MakeShape(F32, filter_dims); { auto input = builder.Parameter(0, input_shape, "input"); auto filter = builder.Parameter(1, filter_shape, "filter"); - // Tensorflow dimension numbers for 2D convolution. + // Tensorflow dimension numbers for 1D convolution. ConvolutionDimensionNumbers dnums; dnums.set_input_batch_dimension(0); dnums.set_output_batch_dimension(0); @@ -538,28 +552,57 @@ XLA_TEST_F(ConvolutionTest, Convolve1D_Valid) { } std::vector input_elems(ShapeUtil::ElementsIn(input_shape), 1.0); - // std::iota(input_elems.begin(), input_elems.end(), 1.0f); auto input_r1 = Literal::CreateR1(input_elems); - auto input_r4 = input_r1->Reshape(input_dims).ConsumeValueOrDie(); + auto input_r3 = input_r1->Reshape(input_dims).ConsumeValueOrDie(); std::vector filter_elems(ShapeUtil::ElementsIn(filter_shape), 1.0); - // std::iota(filter_elems.begin(), filter_elems.end(), 1.0f); auto filter_r1 = Literal::CreateR1(filter_elems); - auto filter_r4 = filter_r1->Reshape(filter_dims).ConsumeValueOrDie(); + auto filter_r3 = filter_r1->Reshape(filter_dims).ConsumeValueOrDie(); - std::vector expect_elems(batch * output_feature * length, 256); + std::vector expect_elems(batch * output_feature * num_windows, + window_size * input_feature); auto expected_r1 = Literal::CreateR1(expect_elems); - auto expected_r4 = - expected_r1->Reshape({batch, length, output_feature}).ConsumeValueOrDie(); + auto expected_r3 = expected_r1->Reshape({batch, num_windows, output_feature}) + .ConsumeValueOrDie(); - auto input_literal = client_->TransferToServer(*input_r4).ConsumeValueOrDie(); + auto input_literal = client_->TransferToServer(*input_r3).ConsumeValueOrDie(); auto filter_literal = - client_->TransferToServer(*filter_r4).ConsumeValueOrDie(); - ComputeAndCompareLiteral(&builder, *expected_r4, + client_->TransferToServer(*filter_r3).ConsumeValueOrDie(); + ComputeAndCompareLiteral(&builder, *expected_r3, {input_literal.get(), filter_literal.get()}, error_spec_); } +INSTANTIATE_TEST_CASE_P( + Convolve1D1WindowTest_Instantiation, Convolve1D1WindowTest, + ::testing::Values(Convolve1DTestParam{1, 1, 1, 1, 2}, + Convolve1DTestParam{160, 1, 1, 5, 1}, + Convolve1DTestParam{24, 1, 1, 20, 1}, + Convolve1DTestParam{30, 1, 1, 20, 1}, + Convolve1DTestParam{23, 1, 1, 20, 20}, + Convolve1DTestParam{25, 1, 1, 20, 1}, + Convolve1DTestParam{24, 1, 1, 10, 5}, + Convolve1DTestParam{160, 1, 1, 10, 1}, + Convolve1DTestParam{255, 1, 1, 3, 1}, + Convolve1DTestParam{130, 1, 1, 1, 3}, + Convolve1DTestParam{64, 1, 1, 1, 1}, + Convolve1DTestParam{128, 1, 1, 1, 1}, + Convolve1DTestParam{139, 1, 1, 128, 1}, + Convolve1DTestParam{1, 10, 10, 1, 10}, + Convolve1DTestParam{1, 10, 130, 1, 2}, + Convolve1DTestParam{1, 10, 130, 1, 1}, + Convolve1DTestParam{1, 64, 64, 1, 10}, + Convolve1DTestParam{1, 65, 65, 1, 1}, + Convolve1DTestParam{1, 128, 128, 1, 1}, + Convolve1DTestParam{128, 128, 128, 128, 1}, + Convolve1DTestParam{1, 128, 128, 1, 1}, + Convolve1DTestParam{2, 2, 2, 2, 1}, + Convolve1DTestParam{161, 1, 1, 10, 1}, + Convolve1DTestParam{900, 1, 1, 10, 1}, + Convolve1DTestParam{640, 3, 3, 128, 1}) + +); + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/tests/custom_call_test.cc b/tensorflow/compiler/xla/tests/custom_call_test.cc index 342478bc744273be9deb8b750b5a6a47b7d9f91b..74f73a1ddc15be033e52b0b45f9961e5dc3a1ecb 100644 --- a/tensorflow/compiler/xla/tests/custom_call_test.cc +++ b/tensorflow/compiler/xla/tests/custom_call_test.cc @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/cpu/custom_call_target_registry.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -31,19 +32,19 @@ limitations under the License. #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/test.h" - -extern "C" void TF_EXPORT R0F32Add2(float* out, float** in) { +namespace { +void R0F32Add2(float* out, float** in) { TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float*)); *out = **in + 2.0f; } -extern "C" void TF_EXPORT R2F32ReduceSum(float* out, float** in) { +void R2F32ReduceSum(float* out, float** in) { TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4); float* array = in[0]; *out = array[0] + array[1] + array[2] + array[3]; } -extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) { +void Add1ToValues(float* out, float** in) { TF_ANNOTATE_MEMORY_IS_INITIALIZED(in, sizeof(float) * 4); float* array = in[0]; out[0] = array[0] + 1; @@ -51,6 +52,11 @@ extern "C" void TF_EXPORT Add1ToValues(float* out, float** in) { out[2] = array[2] + 1; out[3] = array[3] + 1; } +} // namespace + +REGISTER_CUSTOM_CALL_TARGET(R0F32Add2); +REGISTER_CUSTOM_CALL_TARGET(R2F32ReduceSum); +REGISTER_CUSTOM_CALL_TARGET(Add1ToValues); namespace xla { namespace { diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc index 224aa57899d04eb8309b2337bb8fc936a81d350f..cf089d748dcd4f5db637ff9087c5fbc504c82572 100644 --- a/tensorflow/compiler/xla/tests/dot_operation_test.cc +++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc @@ -347,7 +347,7 @@ XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTF) { TestNonsquareMatrixDot(kLhsRowMajor, kRhsRowMajor); } -TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTT) { +XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF32MajorToMinorTT) { constexpr bool kLhsRowMajor = true; constexpr bool kRhsRowMajor = true; TestNonsquareMatrixDot(kLhsRowMajor, kRhsRowMajor); @@ -357,7 +357,11 @@ XLA_TEST_F(DotOperationTest, NonsquareMatrixDotF64) { TestNonsquareMatrixDot(); } -TEST_F(DotOperationTest, ConcurrentMatMul) { +XLA_TEST_F(DotOperationTest, NonsquareMatrixDotC64) { + TestNonsquareMatrixDot(); +} + +XLA_TEST_F(DotOperationTest, ConcurrentMatMul) { ComputationBuilder builder(client_, TestName()); auto matrix1 = builder.ConstantR2({{1.0, 2.0}, {3.0, 4.0}}); auto matrix2 = builder.ConstantR2({{5.0, 6.0}, {7.0, 8.0}}); diff --git a/tensorflow/compiler/xla/tests/fusion_test.cc b/tensorflow/compiler/xla/tests/fusion_test.cc index 3bf9ccb19745b9e91d99614792dbec0443818f2b..a8f6488996087b57e3121ce2c7de918070950c72 100644 --- a/tensorflow/compiler/xla/tests/fusion_test.cc +++ b/tensorflow/compiler/xla/tests/fusion_test.cc @@ -17,8 +17,12 @@ limitations under the License. #include #include #include +#include #include +#define EIGEN_USE_THREADS + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/compiler/xla/array2d.h" #include "tensorflow/compiler/xla/client/client_library.h" #include "tensorflow/compiler/xla/client/computation.h" @@ -37,6 +41,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/literal_test_util.h" #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" @@ -250,6 +255,42 @@ XLA_TEST_F(FusionTest, Parameter) { ErrorSpec(1e-4)); } +XLA_TEST_F(FusionTest, RandomizedParallelPartition) { + // Tests parallel partitioning of a fusion instruction. + // Create shape with random outer dimension size to generate random parallel + // partition counts for each test run. + const int seed = tensorflow::testing::RandomSeed(); + LOG(INFO) << "RandomizedParallelPartition seed: " << seed; + std::mt19937 generator(seed); + std::uniform_int_distribution distribution(128, 1024); + const int64 rand_dim0_size = distribution(generator); + const int64 dim1_size = 1024; + Shape shape = + ShapeUtil::MakeShapeWithLayout(F32, {rand_dim0_size, dim1_size}, {1, 0}); + // Build simple fusion computation: y = x^2 (elementwise). + auto builder = HloComputation::Builder(TestName()); + auto hlo_module = CreateNewModule(); + + auto two = builder.AddInstruction( + HloInstruction::CreateConstant(Literal::CreateR0(2.0))); + auto x = + builder.AddInstruction(HloInstruction::CreateBroadcast(shape, two, {})); + auto y = builder.AddInstruction( + HloInstruction::CreateBinary(shape, HloOpcode::kMultiply, x, x)); + + hlo_module->AddEntryComputation(builder.Build()) + ->CreateFusionInstruction(/*instructions_to_fuse=*/{y, x, two}, + HloInstruction::FusionKind::kLoop); + // Compute result. + auto result = ExecuteAndTransfer(std::move(hlo_module), {}); + // Every element of result should be y = x^2 = 4.0. + for (int i = 0; i < rand_dim0_size; ++i) { + for (int j = 0; j < dim1_size; ++j) { + EXPECT_EQ(4.0, result->Get({i, j})); + } + } +} + XLA_TEST_F(FusionTest, BroadcastIntoBinaryOp) { auto builder = HloComputation::Builder(TestName()); auto hlo_module = CreateNewModule(); @@ -722,47 +763,104 @@ void BM_ParallelFusion(int num_iters) { auto executors = PlatformUtil::GetStreamExecutors(platform).ValueOrDie(); StreamExecutorMemoryAllocator allocator(platform, executors); - const int64 intra_op_parallelism_threads = 16; + const int64 intra_op_parallelism_threads = 24; xla::LocalClientOptions client_options; client_options.set_platform(platform); client_options.set_intra_op_parallelism_threads(intra_op_parallelism_threads); auto client = ClientLibrary::GetOrCreateLocalClient(client_options).ValueOrDie(); - const int64 dim_size = 1024; - // Create a simple fusable elementwise computation. + auto* transfer_manager = + TransferManager::GetForPlatform(platform).ValueOrDie(); + int device_ordinal = client->default_device_ordinal(); + + // Computation shape parameters. + const int64 param0_dim0 = 1024; + const int64 param0_dim1 = 1024; + const int64 param1_dim0 = 1024; + const int64 param1_dim1 = 1024; + const int64 param2_dim0 = 1024; + const int64 param2_dim1 = 1024; + + // Create computation. ComputationBuilder builder(client, "ParallelFusion"); - Shape input_shape = ShapeUtil::MakeShape(F32, {dim_size, dim_size}); - auto input0 = builder.Broadcast(builder.ConstantR0(1.5f), - AsInt64Slice(input_shape.dimensions())); - auto input1 = builder.Broadcast(builder.ConstantR0(2.0f), - AsInt64Slice(input_shape.dimensions())); - auto input2 = builder.Broadcast(builder.ConstantR0(3.0f), - AsInt64Slice(input_shape.dimensions())); - auto x = builder.Mul(input0, input1); - auto y = builder.Add(x, input2); + Shape shape0 = ShapeUtil::MakeShape(F32, {param0_dim0, param0_dim1}); + auto param0 = builder.Parameter(0, shape0, "param0"); + Shape shape1 = ShapeUtil::MakeShape(F32, {param1_dim0, param1_dim1}); + auto param1 = builder.Parameter(1, shape1, "param1"); + Shape shape2 = ShapeUtil::MakeShape(F32, {param2_dim0, param2_dim1}); + auto param2 = builder.Parameter(2, shape2, "param2"); + + auto x = builder.Mul(param0, param1); + auto y = builder.Add(x, param2); auto computation = builder.Build().ConsumeValueOrDie(); + // Transfer literals to device. + auto buffer0 = + ScopedShapedBuffer::Allocate(shape0, &allocator, /*device_ordinal=*/0) + .ConsumeValueOrDie(); + auto param0_literal = + Literal::CreateR2F32Linspace(1.0, 2.0, param0_dim0, param0_dim1); + ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( + executors[device_ordinal], *param0_literal, buffer0->mutable_buffer({}))); + + auto buffer1 = + ScopedShapedBuffer::Allocate(shape1, &allocator, /*device_ordinal=*/0) + .ConsumeValueOrDie(); + auto param1_literal = + Literal::CreateR2F32Linspace(1.0, 2.0, param1_dim0, param1_dim1); + ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( + executors[device_ordinal], *param1_literal, buffer1->mutable_buffer({}))); + + auto buffer2 = + ScopedShapedBuffer::Allocate(shape2, &allocator, /*device_ordinal=*/0) + .ConsumeValueOrDie(); + auto param2_literal = + Literal::CreateR2F32Linspace(1.0, 2.0, param2_dim0, param2_dim1); + ASSERT_IS_OK(transfer_manager->TransferLiteralToDevice( + executors[device_ordinal], *param2_literal, buffer2->mutable_buffer({}))); + + // Build executable. std::unique_ptr executable = - client->Compile(computation, {}, ExecutableBuildOptions()) + client + ->Compile(computation, + {&buffer0->shape(), &buffer1->shape(), &buffer2->shape()}, + ExecutableBuildOptions()) .ConsumeValueOrDie(); - // Run some warm-up executions. + se::Stream stream(executors[client->default_device_ordinal()]); + stream.Init(); + + // Initialize thread pool. + tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), "XLAEigen", + intra_op_parallelism_threads); + tensorflow::EigenThreadPoolWrapper tp(&pool); + Eigen::ThreadPoolDevice device(&tp, tp.NumThreads()); + + // Initialize ExecutableRunOptions. ExecutableRunOptions options; - options.set_allocator(&allocator); + options.set_allocator(&allocator).set_stream(&stream); + options.set_intra_op_thread_pool(&device); + + // Run some warm-up executions. const int kWarmups = 2; for (int i = 0; i < kWarmups; ++i) { - auto result = executable->Run({}, options); + auto result = + executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options); ASSERT_TRUE(result.ok()); } // Run benchmark. - tensorflow::testing::BytesProcessed(static_cast(num_iters) * dim_size * - dim_size * sizeof(float)); + const int64 total_bytes = param0_dim0 * param0_dim0 + + param1_dim0 * param1_dim0 + + param2_dim0 * param2_dim0; + tensorflow::testing::BytesProcessed(static_cast(num_iters) * + total_bytes * sizeof(float)); tensorflow::testing::UseRealTime(); tensorflow::testing::StartTiming(); for (int i = 0; i < num_iters; ++i) { - auto result = executable->Run({}, options); + auto result = + executable->Run({buffer0.get(), buffer1.get(), buffer2.get()}, options); ASSERT_TRUE(result.ok()); } } diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc index 26513d6ce8e0b8896e9f9838ecf28f1ed5bbb383..d73c05ff92578209143e0679558848160cae99bd 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.cc +++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc @@ -19,24 +19,9 @@ limitations under the License. #include #include -#define EIGEN_USE_THREADS - -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h" #include "tensorflow/compiler/xla/ptr_util.h" -#include "tensorflow/compiler/xla/service/backend.h" -#include "tensorflow/compiler/xla/service/computation_layout.h" -#include "tensorflow/compiler/xla/service/executable.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" -#include "tensorflow/compiler/xla/service/hlo_execution_profile.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/transfer_manager.h" -#include "tensorflow/compiler/xla/shape_layout.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/types.h" -#include "tensorflow/core/common_runtime/eigen_thread_pool.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/types.h" @@ -45,22 +30,6 @@ namespace se = ::perftools::gputools; namespace xla { -// Define this in .cc file to avoid having to include eigen or forward declare -// these types in the header. -struct HloTestBase::EigenThreadPoolWrapper { - std::unique_ptr pool; - std::unique_ptr device; -}; - -HloTestBase::HloTestBase() {} - -HloTestBase::~HloTestBase() { - // Deallocate all the memory allocated during the tests. - for (auto& allocation : allocations_) { - backend().default_stream_executor()->Deallocate(&allocation); - } -} - /* static */ std::unique_ptr HloTestBase::CreateNewModule() { HloModuleConfig config; @@ -80,98 +49,25 @@ StatusOr HloTestBase::Execute( tensorflow::gtl::ArraySlice arguments, Shape* result_shape) { - TF_ASSIGN_OR_RETURN( - std::unique_ptr executable, - backend().compiler()->Compile(std::move(module), - backend().default_stream_executor())); - - se::Stream stream(backend().default_stream_executor()); - stream.Init(); - - ExecutableRunOptions run_options; - run_options.set_stream(&stream); - run_options.set_allocator(backend().memory_allocator()); - run_options.set_inter_op_thread_pool(backend().inter_op_thread_pool()); - run_options.set_intra_op_thread_pool( - backend().eigen_intra_op_thread_pool_device()); - - HloExecutionProfile hlo_execution_profile; - ServiceExecutableRunOptions service_run_options( - run_options, backend().StreamBorrower(), - backend().inter_op_thread_pool()); - TF_ASSIGN_OR_RETURN( - se::DeviceMemoryBase result, - executable->ExecuteOnStream(&service_run_options, arguments, - &hlo_execution_profile)); - TF_RET_CHECK(stream.BlockHostUntilDone()); - - allocations_.push_back(result); - - *result_shape = executable->result_shape(); - - if (ShapeUtil::IsTuple(*result_shape)) { - // We must record element buffers of tuples as well to avoid leaks. - DCHECK(!ShapeUtil::IsNestedTuple(*result_shape)); - TF_ASSIGN_OR_RETURN( - std::vector element_buffers, - backend().transfer_manager()->ShallowCopyTupleFromDevice( - backend().default_stream_executor(), result, *result_shape)); - - // A tuple may contain the same buffer in more than one element. Keep track - // of the buffers already added to avoid duplicates in allocations_. - std::set added_opaques; - for (auto element_buffer : element_buffers) { - if (added_opaques.count(element_buffer.opaque()) == 0) { - CHECK(element_buffer.opaque() != nullptr); - added_opaques.insert(element_buffer.opaque()); - allocations_.push_back(element_buffer); - } - } - } - - return result; + return runner_.Execute(std::move(module), arguments, result_shape); } se::DeviceMemoryBase HloTestBase::TransferToDevice(const Literal& literal) { - // Allocate memory on the device using the stream executor. - int64 allocation_size = - backend().transfer_manager()->GetByteSizeRequirement(literal.shape()); - se::DeviceMemoryBase allocation = - backend().default_stream_executor()->AllocateArray( - allocation_size); - allocations_.push_back(allocation); - - TF_CHECK_OK(backend().transfer_manager()->TransferLiteralToDevice( - backend().default_stream_executor(), literal, &allocation)); - - return allocation; + return runner_.TransferToDevice(literal).ValueOrDie(); } std::unique_ptr HloTestBase::TransferFromDevice( const Shape& shape, se::DeviceMemoryBase device_base) { - auto literal = MakeUnique(); - TF_CHECK_OK(backend().transfer_manager()->TransferLiteralFromDevice( - backend().default_stream_executor(), device_base, shape, shape, - literal.get())); - return literal; + return runner_.TransferFromDevice(shape, device_base).ValueOrDie(); } std::unique_ptr HloTestBase::ExecuteAndTransfer( std::unique_ptr module, tensorflow::gtl::ArraySlice arguments) { - Shape result_shape; - se::DeviceMemoryBase device_base = - Execute(std::move(module), arguments, &result_shape).ValueOrDie(); - return TransferFromDevice(result_shape, device_base); + return runner_.ExecuteAndTransfer(std::move(module), arguments).ValueOrDie(); } -Backend& HloTestBase::backend() { - if (!backend_) { - backend_ = Backend::CreateDefaultBackend().ConsumeValueOrDie(); - VLOG(1) << "executing on platform " << backend().platform()->Name(); - } - return *backend_; -} +Backend& HloTestBase::backend() { return runner_.backend(); } /* static */ string HloTestBase::TestName() { diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h index 275f1f5c7baa11245186d119f5b38b4d02b84566..7f068dce36be3546298de2f06bf6d33446d07ca2 100644 --- a/tensorflow/compiler/xla/tests/hlo_test_base.h +++ b/tensorflow/compiler/xla/tests/hlo_test_base.h @@ -21,12 +21,12 @@ limitations under the License. #include #include "tensorflow/compiler/xla/service/backend.h" -#include "tensorflow/compiler/xla/service/compiler.h" -#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/computation_layout.h" #include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_runner.h" +#include "tensorflow/compiler/xla/shape_layout.h" #include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/compiler/xla/tests/literal_test_util.h" -#include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/platform/stream_executor_no_cuda.h" @@ -39,10 +39,9 @@ namespace xla { // building a graph of HLO instructions to run. class HloTestBase : public ::testing::Test { protected: - struct EigenThreadPoolWrapper; - HloTestBase(); + HloTestBase() {} - ~HloTestBase() override; + ~HloTestBase() override {} // Creates a new HLO module for a test. The module created will have // TestName() for its name; it will also automatically populate its debug @@ -102,23 +101,12 @@ class HloTestBase : public ::testing::Test { static string TestName(); - // Creates (if necessary) and returns the default backend. If creation fails, - // crashes the program. - // - // This creates the backend lazily so it's possible to instantiate an - // HloTestBase in a program without any backends linked in. + // Returns the backend owned by the HloRunner. Backend& backend(); - // This vector contains handles of all the device memory allocations performed - // by the test. These are deallocated on destruction of the test object. - std::vector allocations_; + HloRunner runner_; ErrorSpec error_spec_{0.0001}; - - std::unique_ptr thread_pool_wrapper_; - - private: - std::unique_ptr backend_; // Lazily populated. Access via backend(). }; } // namespace xla diff --git a/tensorflow/compiler/xla/tests/literal_test_util.cc b/tensorflow/compiler/xla/tests/literal_test_util.cc index 2876a79dd8b80f5ac1551df4184c853533fb59df..95a52ecd2f5cfc97ec1ccba7d1b7ca6257a8267e 100644 --- a/tensorflow/compiler/xla/tests/literal_test_util.cc +++ b/tensorflow/compiler/xla/tests/literal_test_util.cc @@ -156,6 +156,15 @@ template <> ::testing::AssertionResult CompareEqual(double lhs, double rhs) { return CompareFloatsBitwiseEqual(lhs, rhs); } +template <> +::testing::AssertionResult CompareEqual(complex64 lhs, + complex64 rhs) { + auto res = CompareEqual(lhs.real(), rhs.real()); + if (!res) { + return res; + } + return CompareEqual(lhs.imag(), rhs.imag()); +} // A recursive function which iterates through every index of expected and // actual literal and compares their values elementwise. Returns true if all @@ -235,6 +244,9 @@ bool ExpectLiteralsEqual(const Literal& expected, const Literal& actual, case F64: match = ExpectLiteralsEqual(expected, actual, &multi_index, 0); break; + case C64: + match = ExpectLiteralsEqual(expected, actual, &multi_index, 0); + break; case TUPLE: { bool tuple_match = true; for (int i = 0; i < actual.tuple_literals_size(); ++i) { @@ -325,6 +337,9 @@ class NearComparator { case F64: ExpectLiteralsNear(expected, actual, 0); break; + case C64: + ExpectLiteralsNear(expected, actual, 0); + break; default: LOG(FATAL) << "Unsupported primitive type in near comparator: " << PrimitiveType_Name(expected.shape().element_type()) @@ -365,6 +380,19 @@ class NearComparator { } private: + template + bool NanMismatch(NativeT lhs, NativeT rhs) { + return std::isnan(lhs) != std::isnan(rhs); + } + + template + void ExpectNear(NativeT expected, NativeT actual, + const ::testing::Message& message) { + EXPECT_NEAR(expected, actual, error_.abs) + << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" + << message; + } + // EXPECTs that the two given scalar values are within the error bound. Keeps // track of how many mismatches have occurred to keep the size of the output // manageable. @@ -390,7 +418,7 @@ class NearComparator { "index %s abs_diff %f rel_err %f", LiteralTestUtil::MultiIndexAsString(multi_index_).c_str(), abs_diff, rel_err); - bool nan_mismatch = std::isnan(actual) != std::isnan(expected); + bool nan_mismatch = NanMismatch(expected, actual); bool mismatch = (nan_mismatch || (abs_diff >= error_.abs && rel_err >= error_.rel)); if (mismatch) { @@ -398,11 +426,12 @@ class NearComparator { abs_expected_miscompare_sum_ += std::abs(expected); const int64 kMaxFailures = 2; if (num_miscompares_ < kMaxFailures) { - EXPECT_NEAR(expected, actual, error_.abs) - << "mismatch at index " + ::testing::Message msg; + msg << "mismatch at index " << LiteralTestUtil::MultiIndexAsString(multi_index_) << " abs diff " << abs_diff << " rel err " << rel_err << " failure #" << num_miscompares_; + ExpectNear(expected, actual, msg); } else if (num_miscompares_ == kMaxFailures) { LOG(ERROR) << "reached max 'loud' failure count; silently proceeding..."; @@ -470,6 +499,23 @@ class NearComparator { std::vector max_abs_multi_index_; }; +template <> +bool NearComparator::NanMismatch(complex64 lhs, complex64 rhs) { + return std::isnan(lhs.real()) != std::isnan(rhs.real()) || + std::isnan(lhs.imag()) != std::isnan(rhs.imag()); +} + +template <> +void NearComparator::ExpectNear(complex64 expected, complex64 actual, + const ::testing::Message& message) { + EXPECT_NEAR(expected.real(), actual.real(), error_.abs) + << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" + << message; + EXPECT_NEAR(expected.imag(), actual.imag(), error_.abs) + << "expected:\n " << expected << "\n\tvs actual:\n " << actual << "\n" + << message; +} + } // namespace /* static */ ::testing::AssertionResult LiteralTestUtil::Near( diff --git a/tensorflow/compiler/xla/tests/local_client_execute_test.cc b/tensorflow/compiler/xla/tests/local_client_execute_test.cc index c74213f7f9198741770713aa950e78f2e5ec951d..329b53012f58c8d084cc05f9a567a8aa432c4a3a 100644 --- a/tensorflow/compiler/xla/tests/local_client_execute_test.cc +++ b/tensorflow/compiler/xla/tests/local_client_execute_test.cc @@ -37,6 +37,7 @@ limitations under the License. #include "tensorflow/compiler/xla/tests/test_macros.h" #include "tensorflow/compiler/xla/tests/test_utils.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/stream_executor_no_cuda.h" #include "tensorflow/core/platform/test.h" @@ -859,6 +860,31 @@ XLA_TEST_F(LocalClientExecuteTest, ShapeBufferToLiteralConversion64bit) { Literal::CreateR0(123456789000LL).get()})); } +// TODO(b/34359662): Support infeed/outfeed on GPU and CPU parallel. +// 2017-10-18. +XLA_TEST_F(LocalClientExecuteTest, + DISABLED_ON_GPU(DISABLED_ON_CPU_PARALLEL(InfeedOutfeedTest))) { + ComputationBuilder builder(local_client_, TestName()); + const Shape shape = ShapeUtil::MakeShape(F32, {3}); + auto in = builder.Infeed(shape); + auto constant = builder.ConstantR1({1.0f, 2.0f, 3.0f}); + auto sum = builder.Add(in, constant); + builder.Outfeed(sum, shape, /*outfeed_config=*/""); + + std::unique_ptr thread( + tensorflow::Env::Default()->StartThread( + tensorflow::ThreadOptions(), "execute_thread", + [&] { ExecuteLocallyOrDie(builder.Build().ValueOrDie(), {}); })); + + ASSERT_IS_OK(local_client_->TransferToInfeed( + *Literal::CreateR1({-5.0, 123.0, 42.0}))); + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr result, + local_client_->TransferFromOutfeed(&shape)); + + LiteralTestUtil::ExpectR1Equal({-4.0, 125.0, 45.0}, *result); +} + // Benchmark that measures the overhead of the LocalClient API when running a // trivial computation void BM_LocalClientOverhead(int num_iters) { diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.cc b/tensorflow/compiler/xla/tests/local_client_test_base.cc index 05e282d2081a736fcae1d6a279cdcc37682696f7..c11e1df0a7890a6c3aada5ff47494b42fdaf3b9d 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.cc +++ b/tensorflow/compiler/xla/tests/local_client_test_base.cc @@ -90,6 +90,9 @@ int64 TestAllocator::deallocation_count(int device_ordinal) const { /* static */ TestAllocator* LocalClientTestBase::GetOrCreateAllocator( perftools::gputools::Platform* platform) { + static tensorflow::mutex mu(tensorflow::LINKER_INITIALIZED); + tensorflow::mutex_lock lock(mu); + if (allocator_ == nullptr) { allocator_ = new TestAllocator( platform == nullptr ? PlatformUtil::GetDefaultPlatform().ValueOrDie() diff --git a/tensorflow/compiler/xla/tests/local_client_test_base.h b/tensorflow/compiler/xla/tests/local_client_test_base.h index 17c25adfef9ea2cbe715cd82a199f479e53529b8..3edfcb656ed8278d403103f0cfd820a10892476a 100644 --- a/tensorflow/compiler/xla/tests/local_client_test_base.h +++ b/tensorflow/compiler/xla/tests/local_client_test_base.h @@ -128,8 +128,8 @@ class LocalClientTestBase : public ::testing::Test { return ::testing::UnitTest::GetInstance()->current_test_info()->name(); } - // The allocator must live as long as the service which lives until the end of - // the process, so make the allocator static. + // The allocator must live as long as the service, which lives until the end + // of the process. So make the allocator static. static TestAllocator* allocator_; perftools::gputools::StreamExecutor* stream_executor_; diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc index b48b3a2bdbb0dac3cc7db5f93aa9172dcf47bc02..7bc3185c367f076c9a7d211c9799557e1a91d92f 100644 --- a/tensorflow/compiler/xla/tests/reduce_test.cc +++ b/tensorflow/compiler/xla/tests/reduce_test.cc @@ -457,7 +457,7 @@ XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) { const Shape input_shape = ShapeUtil::MakeShape(F32, {rows, 2, cols / 2}); auto input = builder.Parameter(0, input_shape, "input"); auto zero = builder.ConstantR0(0.0); - auto log_ = builder.Log(input); + auto log_ = builder.Tanh(input); auto reshape = builder.Reshape(log_, {rows, cols}); builder.Reduce(reshape, zero, add_f32, /*dimensions_to_reduce=*/{0}); @@ -473,7 +473,7 @@ XLA_TEST_F(ReduceTest, Reshape_111x2x25Reduce_111x50_To_R1) { for (int64 colno = 0; colno < cols / 2; ++colno) { float column_sum = 0; for (int64 rowno = 0; rowno < rows; ++rowno) { - column_sum += log(input_data(rowno, major, colno)); + column_sum += tanh(input_data(rowno, major, colno)); } expected.push_back(column_sum); } @@ -502,8 +502,8 @@ XLA_TEST_F(ReduceTest, AddReduce2DScalarToR0) { ComputationBuilder builder(client_, TestName()); auto add = CreateScalarAddComputation(F32, &builder); auto scalar = builder.ConstantR0(42.0); - auto broacasted = builder.Broadcast(scalar, {500, 500}); - builder.Reduce(broacasted, builder.ConstantR0(0.0f), add, {0, 1}); + auto broadcasted = builder.Broadcast(scalar, {500, 500}); + builder.Reduce(broadcasted, builder.ConstantR0(0.0f), add, {0, 1}); float expected = 42.0f * static_cast(500 * 500); ComputeAndCompareR0(&builder, expected, {}, ErrorSpec(0.0001)); @@ -514,8 +514,8 @@ XLA_TEST_F(ReduceTest, MaxReduce2DScalarToR0) { ComputationBuilder builder(client_, TestName()); auto max = CreateScalarMaxComputation(F32, &builder); auto scalar = builder.ConstantR0(42.0); - auto broacasted = builder.Broadcast(scalar, {500, 500}); - builder.Reduce(broacasted, builder.ConstantR0(0.0f), max, {0, 1}); + auto broadcasted = builder.Broadcast(scalar, {500, 500}); + builder.Reduce(broadcasted, builder.ConstantR0(0.0f), max, {0, 1}); float expected = 42.0f; ComputeAndCompareR0(&builder, expected, {}, ErrorSpec(0.0001)); diff --git a/tensorflow/compiler/xla/tests/unary_op_test.cc b/tensorflow/compiler/xla/tests/unary_op_test.cc index efae13a43a058b03a45174c8260bce2ed70cb75c..fa4192e9281784a4a3063601afe89fba6a9dac18 100644 --- a/tensorflow/compiler/xla/tests/unary_op_test.cc +++ b/tensorflow/compiler/xla/tests/unary_op_test.cc @@ -41,7 +41,11 @@ class UnaryOpTest : public ClientLibraryTestBase { auto arg = builder.ConstantR1({}); auto abs = builder.Abs(arg); - ComputeAndCompareR1(&builder, {}, {}); + if (primitive_util::NativeToPrimitiveType() == C64) { + ComputeAndCompareR1(&builder, {}, {}); + } else { + ComputeAndCompareR1(&builder, {}, {}); + } } template @@ -80,14 +84,58 @@ int UnaryOpTest::inf() { return 2147483647; } +template <> +void UnaryOpTest::AbsTestHelper() { + ComputationBuilder builder(client_, TestName()); + auto arg = builder.ConstantR1({{-2, 0}, + {0, 25}, + {0, 0}, + {-0.3f, 0.4f}, + {0, inf()}, + {-inf(), 0}}); + auto abs = builder.Abs(arg); + + std::unique_ptr expected = + Literal::CreateR1({2, 25, 0, 0.5, inf(), inf()}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); +} + +template <> +void UnaryOpTest::SignTestHelper() { + ComputationBuilder builder(client_, TestName()); + auto arg = builder.ConstantR1( + {{-2, 0}, {0, 25}, {0, 0}, {static_cast(-0.0), 0}, {-1, 1}}); + auto sign = builder.Sign(arg); + + std::unique_ptr expected = Literal::CreateR1( + {{-1, 0}, {0, 1}, {0, 0}, {0, 0}, {-std::sqrt(0.5f), std::sqrt(0.5f)}}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); +} + +template <> +void UnaryOpTest::SignAbsTestHelper() { + ComputationBuilder builder(client_, TestName()); + auto arg = + builder.ConstantR1({{-2, 0}, {0, 25}, {0, 0}, {-0.4, 0.3}}); + auto sign = builder.Sign(arg); + auto abs = builder.Abs(arg); + builder.Sub(builder.Mul(sign, builder.ConvertElementType(abs, C64)), arg); + + std::unique_ptr expected = + Literal::CreateR1({0, 0, 0, 0}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); +} + XLA_TEST_F(UnaryOpTest, AbsTestR1Size0) { AbsSize0TestHelper(); AbsSize0TestHelper(); + AbsSize0TestHelper(); } XLA_TEST_F(UnaryOpTest, AbsTestR1) { AbsTestHelper(); AbsTestHelper(); + AbsTestHelper(); } XLA_TEST_F(UnaryOpTest, AbsTestR0) { @@ -98,34 +146,44 @@ XLA_TEST_F(UnaryOpTest, AbsTestR0) { auto absf = builder.Abs(argf); auto argf0 = builder.ConstantR0(-0.0f); auto absf0 = builder.Abs(argf0); - builder.Add(absf0, builder.Add(absf, builder.ConvertElementType( - absi, PrimitiveType::F32))); + auto argc = builder.ConstantR0({-0.3f, 0.4f}); + auto absc = builder.Abs(argc); + builder.Add(builder.Add(absc, absf0), + builder.Add(absf, builder.ConvertElementType(absi, F32))); - ComputeAndCompareR0(&builder, 8.0f, {}); + ComputeAndCompareR0(&builder, 8.5f, {}); } XLA_TEST_F(UnaryOpTest, SignTestR0) { ComputationBuilder builder(client_, TestName()); auto argi = builder.ConstantR0(-5); - auto absi = builder.Sign(argi); + auto sgni = builder.Sign(argi); // -1 auto argf = builder.ConstantR0(-4.0f); - auto absf = builder.Sign(argf); + auto sgnf = builder.Sign(argf); // -1 auto argf0 = builder.ConstantR0(-0.0f); - auto absf0 = builder.Sign(argf0); - builder.Add(absf0, builder.Add(absf, builder.ConvertElementType( - absi, PrimitiveType::F32))); - - ComputeAndCompareR0(&builder, -2.0f, {}); + auto sgnf0 = builder.Sign(argf0); // 0 + auto argc = builder.ConstantR0({-.3, .4}); + auto sgnc = builder.Sign(argc); // (-.6, .8) + builder.Add(sgnc, builder.ConvertElementType( + builder.Add(builder.Add(sgnf0, sgnf), + builder.ConvertElementType(sgni, F32)), + C64)); + + std::unique_ptr expected = + Literal::CreateR0({-2.6f, 0.8f}); + ComputeAndCompareLiteral(&builder, *expected, {}, ErrorSpec(1e-6f)); } XLA_TEST_F(UnaryOpTest, SignTestR1) { SignTestHelper(); SignTestHelper(); + SignTestHelper(); } XLA_TEST_F(UnaryOpTest, SignAbsTestR1) { SignAbsTestHelper(); SignAbsTestHelper(); + SignAbsTestHelper(); } XLA_TEST_F(UnaryOpTest, UnsignedAbsTestR1) { diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD index 0451537af777e127df333da8a941a89e6fe315c2..759921dce5acf3cd23a121776f3ab0731c9bb623 100644 --- a/tensorflow/compiler/xla/tools/BUILD +++ b/tensorflow/compiler/xla/tools/BUILD @@ -210,6 +210,18 @@ tf_cc_binary( ], ) +tf_cc_binary( + name = "hlo_proto_to_json", + srcs = ["hlo_proto_to_json.cc"], + deps = [ + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla/service:hlo_proto", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + ], +) + # ----------------------------------------------------------------------------- filegroup( diff --git a/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc b/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc new file mode 100644 index 0000000000000000000000000000000000000000..4e02e17db65c0a4220672733be8319e1a0cc4f0f --- /dev/null +++ b/tensorflow/compiler/xla/tools/hlo_proto_to_json.cc @@ -0,0 +1,91 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Usage: +// hlo_proto_to_json --input_file=some_binary_proto +// --output_file=path_to_dump_output +// +// Reads one serilized Hlo module, convert it into JSON format and dump into +// some output directory. some_binaray_proto is obtained by serializing Hlo +// module to disk using --xla_dump_hlo_proto_to debug optoin. + +#include +#include +#include + +#include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/command_line_flags.h" + +using tensorflow::Env; +using xla::string; + +namespace xla { +namespace tools { + +StatusOr ToJson(const tensorflow::protobuf::Message& message) { + string json_output; + tensorflow::protobuf::util::JsonPrintOptions json_options; + json_options.add_whitespace = true; + json_options.always_print_primitive_fields = true; + auto status = tensorflow::protobuf::util::MessageToJsonString( + message, &json_output, json_options); + if (!status.ok()) { + return InternalError("MessageToJsonString failed: %s", + status.error_message().data()); + } + return json_output; +} + +void RealMain(const string& input, const string& output) { + HloProto hlo_proto; + TF_CHECK_OK(tensorflow::ReadBinaryProto(tensorflow::Env::Default(), input, + &hlo_proto)) + << "Can't open, read, or parse input file " << input; + + auto statusor = ToJson(hlo_proto); + QCHECK(statusor.ok()) << "Error converting " << input << " to JSON." + << statusor.status(); + + TF_CHECK_OK(tensorflow::WriteStringToFile(tensorflow::Env::Default(), output, + statusor.ValueOrDie())); +} + +} // namespace tools +} // namespace xla + +int main(int argc, char** argv) { + string input_file, output_file; + const std::vector flag_list = { + tensorflow::Flag("input_file", &input_file, "file to convert."), + tensorflow::Flag("output_file", &output_file, "converted file"), + }; + const string usage = tensorflow::Flags::Usage(argv[0], flag_list); + bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list); + tensorflow::port::InitMain(usage.c_str(), &argc, &argv); + QCHECK(parse_ok && argc == 1) << "\n" << usage; + + QCHECK(!input_file.empty()) << "--input_file is required"; + QCHECK(!output_file.empty()) << "--output_file is required"; + + xla::tools::RealMain(input_file, output_file); + + return 0; +} diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..c84ca9fc833881ce49bcaad5dd85394145151912 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/BUILD @@ -0,0 +1,84 @@ +# Build file for the Hlo parser. + +licenses(["notice"]) # Apache 2.0 + +package( + default_visibility = [":friends"], +) + +package_group( + name = "friends", + includes = [ + "//tensorflow/compiler/xla:friends", + ], +) + +# Filegroup used to collect source files for dependency checking. +filegroup( + name = "c_srcs", + data = glob([ + "**/*.cc", + "**/*.h", + ]), +) + +load("//tensorflow:tensorflow.bzl", "tf_cc_test") + +cc_library( + name = "hlo_lexer", + srcs = ["hlo_lexer.cc"], + hdrs = [ + "hlo_lexer.h", + "hlo_token.h", + ], + deps = [ + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core:lib", + "//tensorflow/core:regexp_internal", + ], +) + +cc_library( + name = "hlo_parser", + srcs = ["hlo_parser.cc"], + hdrs = ["hlo_parser.h"], + deps = [ + ":hlo_lexer", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + ], +) + +tf_cc_test( + name = "hlo_parser_test", + size = "small", + srcs = ["hlo_parser_test.cc"], + deps = [ + ":hlo_parser", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + +# ----------------------------------------------------------------------------- + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/compiler/xla/tools/parser/README.md b/tensorflow/compiler/xla/tools/parser/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2feaa49db86ea700cab0b794ec441b95ac03b468 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/README.md @@ -0,0 +1,85 @@ +# HloModule string syntax + +TODO: Support all subcomputations (for fusion, reduce, ...). + +TODO: Support all extra attributes, e.g. dimensions, strides. + +```yacc +hlo_module + : 'HloModule' name computations + ; + +computations + : computation + | computation computations + ; + +computation + : 'ENTRY' name param_list '->' shape instruction_list + | name param_list '->' shape instruction_list + ; + +instruction_list + : '{' instruction_list1 '}' + ; +instruction_list1 + : instruction + | instruction_list1 instruction + ; +instruction + : 'ROOT' name '=' shape opcode operands extra_attributes + | name '=' shape opcode operands extra_attributes + ; + +operands + : '(' operands1 ')' + ; +operands1 + : /*empty*/ + | operand + | operands1 ',' operand + ; +operand + : shape name + ; + +extra_attributes + : /*empty*/ + | ',' extra_attribute + | ',' extra_attribute extra_attributes + ; +extra_attribute + : attribute_name attribute_value + ; + +param_list + : '(' param_list1 ')' + ; +param_list1 + : /*empty*/ + | param + | param_list1 ',' param + ; +param + : name shape + ; + +shape + : shape_val_ + | '(' tuple_elements ')' + ; +tuple_elements + : /*empty*/ + | shape (',' shape)* + ; + +name + : identifier ':' + | '%' identifier + ; + +identifier + : [a-zA-Z_][a-zA-Z0-9_.-]* + ; + +``` diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc new file mode 100644 index 0000000000000000000000000000000000000000..486df6854016d2d796781d722e6a6a27273e1cf3 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.cc @@ -0,0 +1,281 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" + +#include + +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/lib/gtl/optional.h" +#include "tensorflow/core/lib/strings/numbers.h" +#include "tensorflow/core/platform/regexp.h" + +namespace xla { +namespace tools { + +using tensorflow::StringPiece; + +namespace { + +constexpr int kEOF = -1; +constexpr int kError = -2; + +// [a-zA-Z0-9_.-] +bool IsIdentifierChar(char c) { + return isalnum(static_cast(c)) || c == '-' || c == '.' || + c == '_'; +} + +} // namespace + +int HloLexer::GetNextChar() { + int current_char = PeekCurrentChar(); + if (current_char != kEOF && current_char != kError) { + current_ptr_++; + } + return current_char; +} + +int HloLexer::PeekCurrentChar() const { + if (current_ptr_ == buf_.end()) { + return kEOF; + } + char current_char = *current_ptr_; + if (current_char == 0) { + // '\0' should not appear in the middle of the string. + return kError; + } + return static_cast(current_char); +} + +bool HloLexer::CanDereference(const char* ptr) const { + return ptr < buf_.end() && ptr >= buf_.begin(); +} + +StringPiece HloLexer::StringPieceFromPointers(const char* begin, + const char* end) const { + CHECK(begin <= end); + CHECK(begin == buf_.end() || CanDereference(begin)); + CHECK(end == buf_.end() || CanDereference(end)); + return StringPiece(begin, end - begin); +} + +tensorflow::RegexpStringPiece HloLexer::RegexpStringPieceFromPointers( + const char* begin, const char* end) const { + CHECK(begin <= end); + CHECK(begin == buf_.end() || CanDereference(begin)); + CHECK(end == buf_.end() || CanDereference(end)); + return tensorflow::RegexpStringPiece(begin, end - begin); +} + +TokKind HloLexer::LexToken() { + while (true) { + token_start_ = current_ptr_; + + int current_char = GetNextChar(); + switch (current_char) { + default: + // [a-zA-Z_] + if (isalpha(static_cast(current_char)) || + current_char == '_') { + return LexIdentifier(); + } + return TokKind::kError; + case kEOF: + // Hit the end of the input buffer. + return TokKind::kEof; + case kError: + // Hit an invalid character in the input buffer. + return TokKind::kError; + case ' ': + case '\t': + case '\n': + case '\r': + // Ignore whitespace. + continue; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '-': + if (current_char == '-' && PeekCurrentChar() == '>') { + current_ptr_++; + return TokKind::kArrow; + } + return LexDigitOrNegative(); + case '=': + return TokKind::kEqual; + case ',': + return TokKind::kComma; + case '%': + return LexPercent(); + case ':': + return TokKind::kColon; + case '[': + return TokKind::kLsquare; + case ']': + return TokKind::kRsquare; + case '{': + return TokKind::kLbrace; + case '}': + return TokKind::kRbrace; + case '(': + return TokKind::kLparen; + case ')': + return TokKind::kRparen; + } + } +} + +// Lex a shape, name, keyword, or opcode. +// shape ::= ([a-zA-Z0-9_]*[0-9]*)\[([0-9,]*)\](?:\s*{([0-9,]*)})? +// name ::= [a-zA-Z_][a-zA-Z0-9_.-]*: +// keyword ::= HloModule, ENTRY, ... +// opcode ::= add, greater-than, ... +// attribute_name ::= condition, body, dimensions, ... +TokKind HloLexer::LexIdentifier() { + { + auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); + // 'consumable' will be advanced iff its prefix matches the pattern. + static LazyRE2 shape_pattern = { + R"(^(\w*\d*)\[([\d,]*)\](?:\s*{([\d,]*)})?)"}; + if (RE2::Consume(&consumable, *shape_pattern)) { + auto status_or_shape = ShapeUtil::ParseShapeString( + StringPieceFromPointers(token_start_, consumable.begin())); + if (status_or_shape.ok()) { + // This is a shape string. + shape_val_ = status_or_shape.ValueOrDie(); + current_ptr_ = consumable.begin(); + return TokKind::kShape; + } + } + } + + while (IsIdentifierChar(PeekCurrentChar())) { + current_ptr_++; + } + + // If followed by ':', it's a name. + if (PeekCurrentChar() == ':') { + str_val_.assign(token_start_, current_ptr_); + current_ptr_++; // skip ':' + return TokKind::kName; + } + + // If followed by '=', it's a attribute name. + if (PeekCurrentChar() == '=') { + str_val_.assign(token_start_, current_ptr_); + current_ptr_++; // skip '=' + return TokKind::kAttributeName; + } + + StringPiece identifier = StringPieceFromPointers(token_start_, current_ptr_); + + // See if this is a keyword. +#define KEYWORD(STR) \ + do { \ + if (identifier == #STR) { \ + return TokKind::kw_##STR; \ + } \ + } while (false) + + KEYWORD(true); + KEYWORD(false); + KEYWORD(HloModule); + KEYWORD(ENTRY); + KEYWORD(ROOT); + KEYWORD(maximal); + KEYWORD(replicated); + +#undef KEYWORD + + // See if this is an opcode. + auto opcode = StringToHloOpcode(identifier.ToString()); + if (opcode.ok()) { + opcode_val_ = opcode.ValueOrDie(); + return TokKind::kOpcode; + } + + current_ptr_ = token_start_ + 1; + return TokKind::kError; +} + +// Lex names after a % character. +// name ::= [a-zA-Z_][a-zA-Z0-9_.-]* +TokKind HloLexer::LexPercent() { + const char* name_start = current_ptr_; + if (isalpha(static_cast(PeekCurrentChar())) || + PeekCurrentChar() == '_') { + current_ptr_++; + while (IsIdentifierChar(PeekCurrentChar())) { + current_ptr_++; + } + str_val_.assign(name_start, current_ptr_); + return TokKind::kName; + } + return TokKind::kError; +} + +// Lex integer and floating-point values. +// int [-]?[0-9]+ +// fp with exp [-]?([0-9]+|[0-9]+[.][0-9]*|[0-9]*[.][0-9]+)([eE][+-]?[0-9]+) +// fp without exp [-]?([0-9]+[.][0-9]*|[0-9]*[.][0-9]+) +TokKind HloLexer::LexDigitOrNegative() { + auto consumable = RegexpStringPieceFromPointers(token_start_, buf_.end()); + static LazyRE2 float_pattern = { + R"([-]?((\d+|\d+[.]\d*|\d*[.]\d+)([eE][+-]?\d+))|(\d+[.]\d*|\d*[.]\d+))"}; + if (RE2::Consume(&consumable, *float_pattern)) { + current_ptr_ = consumable.begin(); + tensorflow::strings::safe_strtod(string(token_start_, current_ptr_).c_str(), + &decimal_val_); + return TokKind::kDecimal; + } + + static LazyRE2 int_pattern = {R"([-]?\d+)"}; + if (RE2::Consume(&consumable, *int_pattern)) { + current_ptr_ = consumable.begin(); + tensorflow::strings::safe_strto64( + StringPieceFromPointers(token_start_, current_ptr_), &int64_val_); + return TokKind::kInt; + } + + return TokKind::kError; +} + +StringPiece HloLexer::GetCurrentLine() const { + const char* start = token_start_; + const char* end = current_ptr_; + if (!CanDereference(start) || !CanDereference(end)) { + return "LINE OUT OF RANGE"; + } + while (start > buf_.begin() && *start != '\n') { + start--; + } + while (end < buf_.end() && *end != '\n') { + end++; + } + return StringPieceFromPointers(start, end); +} + +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_lexer.h b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h new file mode 100644 index 0000000000000000000000000000000000000000..433a3a3601e969de154d2f463f650f5f0b07a49f --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_lexer.h @@ -0,0 +1,113 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ + +#include + +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_token.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/regexp.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { +namespace tools { + +// Lexer for the HloModule::ToString() format text. +class HloLexer { + public: + explicit HloLexer(tensorflow::StringPiece buf) : buf_(buf) { + current_ptr_ = buf_.begin(); + } + + TokKind Lex() { return current_kind_ = LexToken(); } + TokKind GetKind() const { return current_kind_; } + string GetStrVal() const { + switch (GetKind()) { + case TokKind::kName: + case TokKind::kAttributeName: + return str_val_; + default: + LOG(FATAL) << "This token does not have string value"; + } + } + Shape GetShapeVal() const { + CHECK(GetKind() == TokKind::kShape); + return shape_val_; + } + HloOpcode GetOpcodeVal() const { + CHECK(GetKind() == TokKind::kOpcode); + return opcode_val_; + } + int64 GetInt64Val() const { + CHECK(GetKind() == TokKind::kInt); + return int64_val_; + } + double GetDecimalVal() const { + CHECK(GetKind() == TokKind::kDecimal); + return decimal_val_; + } + + // Returns the line of text that is currently being lexed. + tensorflow::StringPiece GetCurrentLine() const; + + private: + // Returns the current character. If it's neither the end of input buffer nor + // an invalid character, moves the pointer forward. + int GetNextChar(); + + // Returns the current character. + int PeekCurrentChar() const; + + // Creates StringPiece with the given begin and end. Exits if the begin > end, + // or it's out of the range of the current buffer. + tensorflow::StringPiece StringPieceFromPointers(const char* begin, + const char* end) const; + tensorflow::RegexpStringPiece RegexpStringPieceFromPointers( + const char* begin, const char* end) const; + + // Returns true if the given ptr is dereferenceable within the range of the + // current buffer. + bool CanDereference(const char* ptr) const; + + TokKind LexToken(); + + TokKind LexIdentifier(); + TokKind LexPercent(); + TokKind LexShape(); + TokKind LexConstant(); + TokKind LexDigitOrNegative(); + + const tensorflow::StringPiece buf_; + const char* current_ptr_; + + // Information about the current token. + const char* token_start_; + TokKind current_kind_; + string str_val_; + Shape shape_val_; + HloOpcode opcode_val_; + int64 int64_val_; + double decimal_val_; +}; + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_LEXER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc new file mode 100644 index 0000000000000000000000000000000000000000..5dd8ec6636ecca6f34fff39f285454ee0764a8ad --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.cc @@ -0,0 +1,828 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" + +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/core/lib/gtl/map_util.h" +#include "tensorflow/core/lib/strings/strcat.h" + +namespace xla { +namespace tools { + +namespace { + +using tensorflow::StringPiece; +using tensorflow::strings::StrCat; + +// Parser for the HloModule::ToString() format text. +class HloParser { + public: + explicit HloParser(StringPiece str) : lexer_(str) {} + + // Runs the parser. Returns false if an error occurred. + bool Run(); + + // Returns the parsed HloModule. + std::unique_ptr ConsumeHloModule() { return std::move(module_); } + + // Returns the error information. + string GetError() const { return tensorflow::str_util::Join(error_, "\n"); } + + private: + // ParseXXX returns false if an error occurred. + bool ParseHloModule(); + bool ParseComputations(); + bool ParseComputation(); + bool ParseInstructionList(HloComputation::Builder* builder, + string* root_name); + bool ParseInstruction(HloComputation::Builder* builder, string* root_name); + bool ParseSharding(HloInstruction* instruction); + bool ParseLiteral(std::unique_ptr* literal, const Shape& shape); + bool ParseOperands(std::vector* operands); + // Fill parsed operands into 'operands' and expect a certain number of + // operands. + bool ParseOperands(std::vector* operands, + const int expected_size); + + template + bool ParseExtraAttribute(T* value, const string& expected_attribute); + template + bool ParseAttributeValue(T* value); + + bool ParseParamList(); + bool ParseName(string* result); + bool ParseAttributeName(string* result); + bool ParseShape(Shape* result); + bool ParseOpcode(HloOpcode* result); + bool ParseInt64(int64* result); + bool ParseDecimal(double* result); + bool ParseBool(bool* result); + bool ParseToken(TokKind kind, const string& msg); + + // Logs the current parsing line and the given message. Always returns false. + bool TokenError(StringPiece msg); + + // If the current token is 'kind', eats it (i.e. lexes the next token) and + // returns true. + bool EatIfPresent(TokKind kind); + + // Adds the instruction to the pool. Returns false and emits an error if the + // instruction already exists. + bool AddInstruction(const string& name, HloInstruction* instruction); + // Adds the computation to the pool. Returns false and emits an error if the + // computation already exists. + bool AddComputation(const string& name, HloComputation* computation); + + // The map from the instruction name to the instruction. This does not own the + // instructions. + std::unordered_map instruction_pool_; + std::unordered_map computation_pool_; + + HloLexer lexer_; + std::unique_ptr module_; + std::vector error_; +}; + +bool HloParser::TokenError(StringPiece msg) { + error_.push_back( + StrCat("was parsing \"", lexer_.GetCurrentLine(), "\"; ", msg)); + return false; +} + +bool HloParser::Run() { + lexer_.Lex(); + return ParseHloModule(); +} + +// ::= 'HloModule' name computations +bool HloParser::ParseHloModule() { + if (lexer_.GetKind() != TokKind::kw_HloModule) { + return TokenError("expects HloModule"); + } + // Eat 'HloModule' + lexer_.Lex(); + + string name; + if (!ParseName(&name)) { + return false; + } + + module_ = MakeUnique(name); + + return ParseComputations(); +} + +// computations ::= (computation)+ +bool HloParser::ParseComputations() { + do { + if (!ParseComputation()) { + return false; + } + } while (lexer_.GetKind() != TokKind::kEof); + return true; +} + +// computation ::= ('ENTRY')? name param_list '->' shape instruction_list +bool HloParser::ParseComputation() { + const bool is_entry_computation = EatIfPresent(TokKind::kw_ENTRY); + string name; + if (!ParseName(&name)) { + return false; + } + auto builder = MakeUnique(name); + + Shape shape; + string root_name; + if (!ParseParamList() || !ParseToken(TokKind::kArrow, "expects '->'") || + !ParseShape(&shape) || !ParseInstructionList(builder.get(), &root_name)) { + return false; + } + + HloInstruction* root = + tensorflow::gtl::FindPtrOrNull(instruction_pool_, root_name); + // This means some instruction was marked as ROOT but we didn't find it in the + // pool, which should not happen. + if (!root_name.empty() && root == nullptr) { + LOG(FATAL) << "instruction " << root_name + << " was marked as ROOT but the parser has not seen it before"; + } + // Now root can be either an existing instruction or a nullptr. If it's a + // nullptr, the implementation of Builder will set the last instruction as + // root instruction. + HloComputation* computation = + is_entry_computation + ? module_->AddEntryComputation(builder->Build(root)) + : module_->AddEmbeddedComputation(builder->Build(root)); + return AddComputation(name, computation); +} + +// instruction_list ::= '{' instruction_list1 '}' +// instruction_list1 ::= (instruction)+ +bool HloParser::ParseInstructionList(HloComputation::Builder* builder, + string* root_name) { + if (!ParseToken(TokKind::kLbrace, + "expects '{' at the beginning of instruction list.")) { + return false; + } + do { + if (!ParseInstruction(builder, root_name)) { + return false; + } + } while (lexer_.GetKind() != TokKind::kRbrace); + return ParseToken(TokKind::kRbrace, + "expects '}' at the end of instruction list."); +} + +// instruction ::= ('ROOT')? name '=' shape opcode operands (extra_attribute)* +bool HloParser::ParseInstruction(HloComputation::Builder* builder, + string* root_name) { + string name; + Shape shape; + HloOpcode opcode; + std::vector operands; + bool is_root = EatIfPresent(TokKind::kw_ROOT); + if (!ParseName(&name) || + !ParseToken(TokKind::kEqual, "expects '=' in instruction") || + !ParseShape(&shape) || !ParseOpcode(&opcode)) { + return false; + } + if (is_root) { + *root_name = name; + } + HloInstruction* instruction; + switch (opcode) { + case HloOpcode::kParameter: { + int64 parameter_number; + if (!ParseToken(TokKind::kLparen, + "expects '(' before parameter number") || + !ParseInt64(¶meter_number) || + !ParseToken(TokKind::kRparen, "expects ')' after parameter number")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateParameter(parameter_number, shape, name)); + break; + } + case HloOpcode::kConstant: { + std::unique_ptr literal; + if (!ParseToken(TokKind::kLparen, + "expects '(' before constant literal") || + !ParseLiteral(&literal, shape) || + !ParseToken(TokKind::kRparen, "expects ')' after constant literal")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateConstant(std::move(literal))); + break; + } + // Unary ops. + case HloOpcode::kAbs: + case HloOpcode::kRoundNearestAfz: + case HloOpcode::kBitcast: + case HloOpcode::kCeil: + case HloOpcode::kCopy: + case HloOpcode::kCos: + case HloOpcode::kExp: + case HloOpcode::kImag: + case HloOpcode::kIsFinite: + case HloOpcode::kFloor: + case HloOpcode::kLog: + case HloOpcode::kNot: + case HloOpcode::kNegate: + case HloOpcode::kReal: + case HloOpcode::kSign: + case HloOpcode::kSin: + case HloOpcode::kSort: + case HloOpcode::kTanh: { + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateUnary(shape, opcode, operands[0])); + break; + } + // Binary ops. + case HloOpcode::kAdd: + case HloOpcode::kDivide: + case HloOpcode::kMultiply: + case HloOpcode::kSubtract: + case HloOpcode::kAtan2: + case HloOpcode::kComplex: + case HloOpcode::kEq: + case HloOpcode::kGe: + case HloOpcode::kGt: + case HloOpcode::kLe: + case HloOpcode::kLt: + case HloOpcode::kNe: + case HloOpcode::kDot: + case HloOpcode::kMaximum: + case HloOpcode::kMinimum: + case HloOpcode::kPower: + case HloOpcode::kRemainder: + case HloOpcode::kAnd: + case HloOpcode::kOr: + case HloOpcode::kShiftLeft: + case HloOpcode::kShiftRightArithmetic: + case HloOpcode::kShiftRightLogical: { + if (!ParseOperands(&operands, /*expected_size=*/2)) { + return false; + } + instruction = builder->AddInstruction(HloInstruction::CreateBinary( + shape, opcode, operands[0], operands[1])); + break; + } + // Ternary ops. + case HloOpcode::kClamp: + case HloOpcode::kSelect: { + if (!ParseOperands(&operands, /*expected_size=*/3)) { + return false; + } + instruction = builder->AddInstruction(HloInstruction::CreateTernary( + shape, opcode, operands[0], operands[1], operands[2])); + break; + } + // Other supported ops. + case HloOpcode::kConvert: { + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateConvert(shape, operands[0])); + break; + } + case HloOpcode::kCrossReplicaSum: { + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateCrossReplicaSum(shape, operands[0])); + break; + } + case HloOpcode::kReshape: { + if (!ParseOperands(&operands, /*expected_size=*/1)) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateReshape(shape, operands[0])); + break; + } + case HloOpcode::kTuple: { + if (!ParseOperands(&operands)) { + return false; + } + instruction = + builder->AddInstruction(HloInstruction::CreateTuple(operands)); + break; + } + case HloOpcode::kWhile: { + HloComputation* condition; + HloComputation* body; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseExtraAttribute(&condition, + /*expected_attribute=*/"condition") || + !ParseExtraAttribute(&body, /*expected_attribute=*/"body")) { + return false; + } + instruction = builder->AddInstruction(HloInstruction::CreateWhile( + shape, condition, body, /*init=*/operands[0])); + break; + } + case HloOpcode::kRecv: { + int64 channel_id; + if (!ParseOperands(&operands, /*expected_size=*/0) || + !ParseExtraAttribute(&channel_id, + /*expected_attribute=*/"channel_id")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateRecv(shape, channel_id)); + break; + } + case HloOpcode::kSend: { + int64 channel_id; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseExtraAttribute(&channel_id, + /*expected_attribute=*/"channel_id")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateSend(operands[0], channel_id)); + break; + } + case HloOpcode::kGetTupleElement: { + int64 index; + if (!ParseOperands(&operands, /*expected_size=*/1) || + !ParseExtraAttribute(&index, /*expected_attribute=*/"index")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateGetTupleElement(shape, operands[0], index)); + break; + } + case HloOpcode::kCall: { + HloComputation* to_apply; + if (!ParseOperands(&operands) || + !ParseExtraAttribute(&to_apply, + /*expected_attribute=*/"to_apply")) { + return false; + } + instruction = builder->AddInstruction( + HloInstruction::CreateCall(shape, operands, to_apply)); + break; + } + case HloOpcode::kBroadcast: + case HloOpcode::kCustomCall: + case HloOpcode::kConcatenate: + case HloOpcode::kReducePrecision: + case HloOpcode::kConvolution: + case HloOpcode::kMap: + case HloOpcode::kPad: + case HloOpcode::kReduce: + case HloOpcode::kReduceWindow: + case HloOpcode::kSelectAndScatter: + case HloOpcode::kReverse: + case HloOpcode::kRng: + case HloOpcode::kSlice: + case HloOpcode::kDynamicSlice: + case HloOpcode::kDynamicUpdateSlice: + case HloOpcode::kTranspose: + case HloOpcode::kFusion: + case HloOpcode::kBatchNormTraining: + case HloOpcode::kBatchNormInference: + case HloOpcode::kInfeed: + case HloOpcode::kOutfeed: + case HloOpcode::kBatchNormGrad: + case HloOpcode::kTrace: + return TokenError(StrCat("parsing not yet implemented for op: ", + HloOpcodeString(opcode))); + } + // Parse "sharding=". + if (lexer_.GetKind() == TokKind::kComma) { + if (!ParseSharding(instruction)) { + return false; + } + } + + return AddInstruction(name, instruction); +} + +// ::= '{' 'replicated'? 'maximal'? ('device=' int)? shape? ('devices=' ('[' +// dims ']')* device_list)? '}' dims ::= int_list device_list ::= int_list +bool HloParser::ParseSharding(HloInstruction* instruction) { + if (!ParseToken(TokKind::kComma, + "expects ',' in front of an extra attribute")) { + return false; + } + string attribute_name; + if (!ParseAttributeName(&attribute_name) || attribute_name != "sharding") { + return TokenError("expects attribute name: sharding"); + } + + if (!ParseToken(TokKind::kLbrace, + "expected '{' to start sharding attribute")) { + return false; + } + + bool maximal = false; + bool replicated = false; + std::vector devices; + std::vector tile_assignment_dimensions; + Shape tile_shape; + while (lexer_.GetKind() != TokKind::kRbrace) { + switch (lexer_.GetKind()) { + case TokKind::kw_maximal: + maximal = true; + lexer_.Lex(); + break; + case TokKind::kw_replicated: + replicated = true; + lexer_.Lex(); + break; + case TokKind::kAttributeName: { + if (lexer_.GetStrVal() == "device") { + if (lexer_.Lex() != TokKind::kInt) { + return TokenError("device= attribute must be an integer"); + } + devices = {lexer_.GetInt64Val()}; + lexer_.Lex(); + } else if (lexer_.GetStrVal() == "devices") { + lexer_.Lex(); + if (!ParseToken(TokKind::kLsquare, + "expected '[' to start sharding devices shape")) { + return false; + } + + do { + int64 dim; + if (!ParseInt64(&dim)) { + return false; + } + tile_assignment_dimensions.push_back(dim); + } while (EatIfPresent(TokKind::kComma)); + + if (!ParseToken(TokKind::kRsquare, + "expected ']' to start sharding devices shape")) { + return false; + } + do { + int64 device; + if (!ParseInt64(&device)) { + return false; + } + devices.push_back(device); + } while (EatIfPresent(TokKind::kComma)); + } else { + return TokenError( + "unknown attribute in sharding: expected device= or devices="); + } + break; + } + case TokKind::kShape: + tile_shape = lexer_.GetShapeVal(); + lexer_.Lex(); + break; + case TokKind::kRbrace: + break; + default: + return TokenError("unexpected token"); + } + } + + OpSharding sharding; + if (replicated) { + if (!devices.empty()) { + return TokenError( + "replicated shardings should not have any devices assigned"); + } + if (!ShapeUtil::Equal(tile_shape, Shape())) { + return TokenError( + "replicated shardings should not have any tile shape set"); + } + sharding.set_type(OpSharding::Type::OpSharding_Type_REPLICATED); + } else if (maximal) { + if (devices.size() != 1) { + return TokenError( + "maximal shardings should have exactly one device assigned"); + } + if (!ShapeUtil::Equal(tile_shape, Shape())) { + return TokenError("maximal shardings should not have any tile shape set"); + } + sharding.set_type(OpSharding::Type::OpSharding_Type_MAXIMAL); + sharding.add_tile_assignment_devices(devices[0]); + } else { + if (devices.size() <= 1) { + return TokenError( + "non-maximal shardings must have more than one device assigned"); + } + if (ShapeUtil::Equal(tile_shape, Shape())) { + return TokenError("non-maximal shardings should have a tile shape set"); + } + if (tile_assignment_dimensions.empty()) { + return TokenError( + "non-maximal shardings must have a tile assignment list including " + "dimensions"); + } + sharding.set_type(OpSharding::Type::OpSharding_Type_OTHER); + *sharding.mutable_tile_shape() = tile_shape; + for (int64 dim : tile_assignment_dimensions) { + sharding.add_tile_assignment_dimensions(dim); + } + for (int64 device : devices) { + sharding.add_tile_assignment_devices(device); + } + } + + instruction->set_sharding(HloSharding::FromProto(sharding).ValueOrDie()); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseLiteral(std::unique_ptr* literal, + const Shape& shape) { + switch (shape.element_type()) { + case PRED: + bool b; + if (!ParseBool(&b)) { + return false; + } + *literal = Literal::CreateR0(b); + return true; + case S32: + int64 i; + if (!ParseInt64(&i)) { + return false; + } + *literal = Literal::CreateR0(i); + return true; + case F32: + double d; + if (!ParseDecimal(&d)) { + return false; + } + *literal = Literal::CreateR0(d); + return true; + default: + return TokenError(StrCat("unsupported constant in shape: ", + ShapeUtil::HumanString(shape))); + } +} + +// operands ::= '(' operands1 ')' +// operands1 +// ::= /*empty*/ +// ::= operand (, operand)* +// operand ::= shape name +bool HloParser::ParseOperands(std::vector* operands) { + if (!ParseToken(TokKind::kLparen, + "expects '(' at the beginning of operands")) { + return false; + } + if (lexer_.GetKind() == TokKind::kRparen) { + // empty + } else { + do { + Shape shape; + string name; + if (!ParseShape(&shape) || !ParseName(&name)) { + return false; + } + HloInstruction* instruction = + tensorflow::gtl::FindPtrOrNull(instruction_pool_, name); + if (!instruction) { + return TokenError(StrCat("instruction does not exist: ", name)); + } + operands->push_back(instruction); + } while (EatIfPresent(TokKind::kComma)); + } + return ParseToken(TokKind::kRparen, "expects ')' at the end of operands"); +} + +bool HloParser::ParseOperands(std::vector* operands, + const int expected_size) { + if (!ParseOperands(operands)) { + return false; + } + if (expected_size != operands->size()) { + return TokenError(StrCat("expects ", expected_size, " operands, but has ", + operands->size(), " operands")); + } + return true; +} + +// extra_attribute ::= ',' attribute_name value +template +bool HloParser::ParseExtraAttribute(T* value, + const string& expected_attribute) { + if (!ParseToken(TokKind::kComma, + "expects ',' in front of an extra attribute")) { + return false; + } + string attribute_name; + if (!ParseAttributeName(&attribute_name) && + attribute_name != expected_attribute) { + return TokenError(StrCat("expects attribute name: ", expected_attribute)); + } + if (!ParseAttributeValue(value)) { + return TokenError( + StrCat("expects value for attribute: ", expected_attribute)); + } + return true; +} + +template <> +bool HloParser::ParseAttributeValue(HloComputation** value) { + string name; + if (!ParseName(&name)) { + return TokenError("expects computation name"); + } + *value = tensorflow::gtl::FindPtrOrNull(computation_pool_, name); + if (*value == nullptr) { + return TokenError(StrCat("computation does not exist: ", name)); + } + return true; +} + +template <> +bool HloParser::ParseAttributeValue(int64* value) { + return ParseInt64(value); +} + +// param_list ::= '(' param_list1 ')' +// param_list1 +// ::= /*empty*/ +// ::= param (',' param)* +// param ::= name shape +bool HloParser::ParseParamList() { + if (!ParseToken(TokKind::kLparen, + "expects '(' at the beginning of param list")) { + return false; + } + + if (lexer_.GetKind() == TokKind::kRparen) { + // empty + } else { + do { + Shape shape; + if (!ParseToken(TokKind::kName, "expects name in parameter") || + !ParseShape(&shape)) { + return false; + } + } while (EatIfPresent(TokKind::kComma)); + } + return ParseToken(TokKind::kRparen, "expects ')' at the end of param list"); +} + +// shape ::= shape_val_ +// shape ::= '(' tuple_elements ')' +// tuple_elements +// ::= /*empty*/ +// ::= shape (',' shape)* +bool HloParser::ParseShape(Shape* result) { + if (EatIfPresent(TokKind::kLparen)) { // Tuple + std::vector shapes; + if (lexer_.GetKind() == TokKind::kRparen) { + /*empty*/ + } else { + // shape (',' shape)* + do { + shapes.emplace_back(); + if (!ParseShape(&shapes.back())) { + return false; + } + } while (EatIfPresent(TokKind::kComma)); + } + *result = ShapeUtil::MakeTupleShape(shapes); + return ParseToken(TokKind::kRparen, "expects ')' at the end of tuple."); + } + + if (lexer_.GetKind() != TokKind::kShape) { + return TokenError("expects shape"); + } + *result = lexer_.GetShapeVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseName(string* result) { + VLOG(1) << "ParseName"; + if (lexer_.GetKind() != TokKind::kName) { + return TokenError("expects name"); + } + *result = lexer_.GetStrVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseAttributeName(string* result) { + if (lexer_.GetKind() != TokKind::kAttributeName) { + return TokenError("expects attribute name"); + } + *result = lexer_.GetStrVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseOpcode(HloOpcode* result) { + VLOG(1) << "ParseOpcode"; + if (lexer_.GetKind() != TokKind::kOpcode) { + return TokenError("expects opcode"); + } + *result = lexer_.GetOpcodeVal(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseInt64(int64* result) { + VLOG(1) << "ParseInt64"; + if (lexer_.GetKind() != TokKind::kInt) { + return TokenError("expects integer"); + } + *result = lexer_.GetInt64Val(); + lexer_.Lex(); + return true; +} + +bool HloParser::ParseDecimal(double* result) { + switch (lexer_.GetKind()) { + case TokKind::kDecimal: + *result = lexer_.GetDecimalVal(); + break; + case TokKind::kInt: + *result = static_cast(lexer_.GetInt64Val()); + break; + default: + return TokenError("expects decimal or integer"); + } + lexer_.Lex(); + return true; +} + +bool HloParser::ParseBool(bool* result) { + if (lexer_.GetKind() != TokKind::kw_true && + lexer_.GetKind() != TokKind::kw_false) { + return TokenError("expects true or false"); + } + *result = lexer_.GetKind() == TokKind::kw_true; + lexer_.Lex(); + return true; +} + +bool HloParser::ParseToken(TokKind kind, const string& msg) { + if (lexer_.GetKind() != kind) { + return TokenError(msg); + } + lexer_.Lex(); + return true; +} + +bool HloParser::EatIfPresent(TokKind kind) { + if (lexer_.GetKind() != kind) { + return false; + } + lexer_.Lex(); + return true; +} + +bool HloParser::AddInstruction(const string& name, + HloInstruction* instruction) { + auto result = instruction_pool_.insert({name, instruction}); + if (!result.second) { + return TokenError(StrCat("instruction already exists: ", name)); + } + return true; +} + +bool HloParser::AddComputation(const string& name, + HloComputation* computation) { + auto result = computation_pool_.insert({name, computation}); + if (!result.second) { + return TokenError(StrCat("computation already exists: ", name)); + } + return true; +} + +} // namespace + +StatusOr> Parse(StringPiece str) { + HloParser parser(str); + if (!parser.Run()) { + return InvalidArgument("Syntax error: %s", parser.GetError().c_str()); + } + return parser.ConsumeHloModule(); +} + +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser.h b/tensorflow/compiler/xla/tools/parser/hlo_parser.h new file mode 100644 index 0000000000000000000000000000000000000000..9aaf18ef20d769cd9ac6f0e48bc92f62292ba31a --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser.h @@ -0,0 +1,37 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ + +#include "tensorflow/compiler/xla/ptr_util.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/tools/parser/hlo_lexer.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" + +namespace xla { +namespace tools { + +// The api of the hlo parser. Given a string in the HloModule::ToString() +// format, returns the parsed HloModule. +StatusOr> Parse(tensorflow::StringPiece str); + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_PARSER_H_ diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..5be4d6a2cb1b09355e09e25a40e8dc88bae01650 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc @@ -0,0 +1,321 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h" + +#include +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace xla { +namespace tools { +namespace { + +struct TestData { + string test_name; + string module_string; +}; + +string TestDataToString(const ::testing::TestParamInfo& data) { + return data.param.test_name; +} + +std::vector CreateTestCases() { + // clang-format off + return std::vector({ +// ax + y +{ +"AxpyParam", +R"(HloModule axpy_module: + +ENTRY %axpy.v5 (alpha: f32[2,4], x: f32[2,4], y: f32[2,4]) -> f32[2,4] { + %alpha = f32[2,4]{1,0} parameter(0) + %x = f32[2,4]{1,0} parameter(1) + %multiply = f32[2,4]{1,0} multiply(f32[2,4]{1,0} %alpha, f32[2,4]{1,0} %x) + %y = f32[2,4]{1,0} parameter(2) + ROOT %add = f32[2,4]{1,0} add(f32[2,4]{1,0} %multiply, f32[2,4]{1,0} %y) +} + +)" +}, +// pred constant +{ +"ConstantPred", +R"(HloModule constant_pred_module: + +ENTRY %constant_pred () -> pred[] { + ROOT %constant = pred[] constant(true) +} + +)" +}, +// s32 constant +{ +"ConstantS32", +R"(HloModule constant_s32_module: + +ENTRY %constant_s32 () -> s32[] { + ROOT %constant = s32[] constant(-42) +} + +)" +}, +// f32 constant, but the value is not a decimal +{ +"ConstantF32", R"(HloModule ConstantF32_module: + +ENTRY %ConstantF32.v4 () -> f32[] { + ROOT %constant = f32[] constant(42) +} + +)" +}, +// constant + constant +{ +"AddConstants", +R"(HloModule add_constants_module: + +ENTRY %add_constants () -> f32[] { + %constant = f32[] constant(3.14) + ROOT %add = f32[] add(f32[] %constant, f32[] %constant) +} + +)" +}, +// v1 > v2 ? v1 : v2 +{ +"SelectR1F32", +R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module: + +ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { + %v1 = f32[4]{0} parameter(0), sharding={maximal device=1} + %v2 = f32[4]{0} parameter(1), sharding={maximal device=1} + %greater-than = pred[4]{0} greater-than(f32[4]{0} %v1, f32[4]{0} %v2), sharding={replicated} + ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2) +} + +)" +}, +// empty tuple +{ +"EmptyTupleCreate", +R"(HloModule EmptyTupleCreate_module: + +ENTRY %EmptyTupleCreate.v1 () -> () { + ROOT %tuple = () tuple() +} + +)" +}, +// tuple +{ +"TupleCreate", +R"(HloModule TupleCreate_module: + +ENTRY %TupleCreate.v4 (v1: f32[], v2: f32[3], v3: f32[2,3]) -> (f32[], f32[3], f32[2,3]) { + %v1 = f32[] parameter(0) + %v2 = f32[3]{0} parameter(1) + %v3 = f32[2,3]{1,0} parameter(2) + ROOT %tuple = (f32[], f32[3]{0}, f32[2,3]{1,0}) tuple(f32[] %v1, f32[3]{0} %v2, f32[2,3]{1,0} %v3) +} + +)" +}, +// int32 result = 0; +// while (result < 5) { result = result + 1; } +{ +"WhileWithScalarS32Result", +R"(HloModule WhileWithScalarS32Result_module: + +%body.v3 (prev.1: s32[]) -> s32[] { + %constant = s32[] constant(1) + %prev.1 = s32[] parameter(0) + ROOT %add = s32[] add(s32[] %constant, s32[] %prev.1) +} + +%condition.v3 (prev.2: s32[]) -> pred[] { + %constant.1 = s32[] constant(5) + %prev.2 = s32[] parameter(0) + ROOT %greater-than = pred[] greater-than(s32[] %constant.1, s32[] %prev.2) +} + +ENTRY %WhileWithScalarS32Result.v2 () -> s32[] { + %constant.2 = s32[] constant(0) + ROOT %while = s32[] while(s32[] %constant.2), condition=%condition.v3, body=%body.v3 +} + +)" +}, +// send and recv +{ +"SendRecv", +R"(HloModule TwoSendRecvBothWayRecvFist_module: + +ENTRY %TwoSendRecvBothWayRecvFist.v3 () -> f32[] { + %recv = f32[] recv(), channel_id=15, sharding={maximal device=1} + ROOT %constant = f32[] constant(2.1), sharding={maximal device=0} + %send = () send(f32[] %constant), channel_id=16, sharding={maximal device=0} +} + +)" +}, +// get-tuple-element +{ +"GetTupleElement", +R"(HloModule GetTupleElement_module: + +ENTRY %GetTupleElement.v4 () -> s32[] { + %constant = f32[] constant(1.23) + %constant.1 = s32[] constant(4) + %tuple = (f32[], s32[]) tuple(f32[] %constant, s32[] %constant.1) + ROOT %get-tuple-element = s32[] get-tuple-element((f32[], s32[]) %tuple), index=1, sharding={maximal device=0} +} + +)" +}, +// call +{ +"Call", +R"(HloModule CallR0F32IdentityScalar_module: + +%Identity.v1 (x: f32[]) -> f32[] { + ROOT %x = f32[] parameter(0) +} + +ENTRY %CallR0F32IdentityScalar.v2 () -> f32[] { + %constant = f32[] constant(42) + ROOT %call = f32[] call(f32[] %constant), to_apply=%Identity.v1 +} + +)" +} + }); + // clang-format on +} + +class HloParserTest : public ::testing::Test, + public ::testing::WithParamInterface { + protected: + void ExpectSuccess() { + const string& original = GetParam().module_string; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + EXPECT_EQ(original, result.ValueOrDie()->ToString()); + } +}; + +TEST_P(HloParserTest, Run) { ExpectSuccess(); } + +INSTANTIATE_TEST_CASE_P(HloParserTestSuccessInstantiation, HloParserTest, + ::testing::ValuesIn(CreateTestCases()), + TestDataToString); + +TEST_F(HloParserTest, Empty) { + const string original = ""; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, Garbage) { + const string original = "HloModule thi$ str1ng makes# N0 sen$e @all!*&^%$"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongOpcode) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: f32[], y: f32[]) -> f32[] { + %x = f32[]{} parameter(0) + %y = f32[]{} parameter(1) + %le = pred[]{} le(f32[]{} %x, f32[]{} %y) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongShape) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: g32[]) -> g32[] { + %x = g32[]{} parameter(0) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, WrongOperandsSize) { + const string original = R"(HloModule wrong_opcode: + +ENTRY %blabla (x: f32[]) -> pred[] { + %x = f32[]{} parameter(0) + %eq = pred[]{} equal-to(f32[]{} %x) +} + +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, OperandNotFound) { + const string original = R"(HloModule operand_not_found: +ENTRY %blabla (x: f32[]) -> pred[] { + %x = f32[]{} parameter(0) + %eq = pred[]{} equal-to(f32[]{} %x, f32[]{} %y) +} +)"; + auto result = Parse(original); + EXPECT_NE(tensorflow::Status::OK(), result.status()); +} + +TEST_F(HloParserTest, MoreConstants) { + const string original = R"(HloModule SelectScalarS32True_module: + +ENTRY %SelectScalarS32True.v4 () -> s32[] { + %constant.2 = pred[] constant(true) + %constant.1 = s32[] constant(-42), sharding={s32[5,6] devices=[2,3]1,2,3,4} + %constant = s32[] constant(42) + %select = s32[] select(pred[] %constant.2, s32[] %constant.1, s32[] %constant) +} + +)"; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + // Constant instructions have no name. The string will be parsed successfully + // but the constant names will not be exactly the same. +} + +TEST_F(HloParserTest, ConstantWithExp) { + const string original = R"(HloModule ConstantWithExp_module: + +ENTRY %ConstantWithExp.v4 () -> f32[] { + %constant.1 = f32[] constant(3e+2) +} + +)"; + auto result = Parse(original); + TF_EXPECT_OK(result.status()); + // The string will be parsed successfully but the output strings are not + // exactly the same, because "3e2" is parsed into value 300 and will be + // printed as "300". +} + +} // namespace +} // namespace tools +} // namespace xla diff --git a/tensorflow/compiler/xla/tools/parser/hlo_token.h b/tensorflow/compiler/xla/tools/parser/hlo_token.h new file mode 100644 index 0000000000000000000000000000000000000000..a40300e2bf0d3279967826be6bf74875f8320f11 --- /dev/null +++ b/tensorflow/compiler/xla/tools/parser/hlo_token.h @@ -0,0 +1,62 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ +#define TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ + +namespace xla { +namespace tools { + +// Defines different kinds of tokens in a hlo module string. +enum class TokKind { + // Markers + kEof, + kError, + + // Tokens with no info. + kEqual, // = + kComma, // , + kColon, // : + kLsquare, + kRsquare, // [ ] + kLbrace, + kRbrace, // { } + kLparen, + kRparen, // ( ) + + kArrow, // -> + + // Keywords + kw_HloModule, + kw_ENTRY, + kw_ROOT, + kw_true, + kw_false, + kw_maximal, + kw_replicated, + + // Typed tokens. + kName, // %foo + kAttributeName, // dimensions= + kShape, // f32[2,3]{1,0} + kOpcode, // add + kInt, // 42 + kDecimal, // 4.2 +}; + +} // namespace tools +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_TOOLS_PARSER_HLO_TOKEN_H_ diff --git a/tensorflow/compiler/xla/types.h b/tensorflow/compiler/xla/types.h index ea8b4b7b989b72034f33920a7d8c1a75e15a7dd1..3b19ca321cad35aad18f7f498e08fd744ffbc371 100644 --- a/tensorflow/compiler/xla/types.h +++ b/tensorflow/compiler/xla/types.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_XLA_TYPES_H_ #define TENSORFLOW_COMPILER_XLA_TYPES_H_ +#include + #include "third_party/eigen3/Eigen/Core" #include "tensorflow/core/platform/types.h" @@ -35,6 +37,8 @@ using ::tensorflow::uint16; using ::tensorflow::uint32; using ::tensorflow::uint64; +using complex64 = std::complex; + using ::Eigen::half; } // namespace xla diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h index f6c0bd1563f4d9090df94b6edd8226119194c76c..f58f57b44396c90a3820835a3d0ecc792aaa7cd0 100644 --- a/tensorflow/compiler/xla/util.h +++ b/tensorflow/compiler/xla/util.h @@ -24,6 +24,7 @@ limitations under the License. #include #include "tensorflow/compiler/xla/status.h" +#include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/types.h" #include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/lib/core/status.h" diff --git a/tensorflow/compiler/xla/xla.bzl b/tensorflow/compiler/xla/xla.bzl index 22e70ec97adf9297ceb3f98f57feb17ae9dafc3d..3fa5bcc1df4f0294582b6c74735fef08c87433eb 100644 --- a/tensorflow/compiler/xla/xla.bzl +++ b/tensorflow/compiler/xla/xla.bzl @@ -17,11 +17,3 @@ def xla_proto_library(name, srcs=[], deps=[], visibility=None, testonly=0): protoc="@protobuf_archive//:protoc", testonly=testonly, visibility=visibility,) - -# Flags required for modules that export symbols that are to be called by the -# XLA CustomCall operator. CustomCall must be able to find symbols with dlsym(), -# which on Linux requires we link with --export-dynamic. -export_dynamic_linkopts = select({ - "//tensorflow:darwin": [], - "//conditions:default": ["-Wl,--export-dynamic"], -}) diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto index 7f4bd26d1bcc3ff9cc002adb28d2adfcf96f59ab..ce3c3eee68ad7f7ebb42836e3cae14803f8650d7 100644 --- a/tensorflow/compiler/xla/xla.proto +++ b/tensorflow/compiler/xla/xla.proto @@ -82,8 +82,8 @@ message DebugOptions { // Dump all HLO modules as text into the provided directory path. string xla_generate_hlo_text_to = 7; - // Dump compilation artifacts as JSON into this directory. - string xla_dump_debug_json_to = 8; + // Dump compilation artifacts in binary proto into this directory. + string xla_dump_hlo_proto_to = 8; // Instrument the computation to collect per-HLO cycle counts. bool xla_hlo_profile = 9; diff --git a/tensorflow/compiler/xla/xla_data.proto b/tensorflow/compiler/xla/xla_data.proto index 0d7e583bedf3e7d942f9e6057679272d12e39fa2..080e3c4267a2dca2b70c5cff51126cbf4b3e2881 100644 --- a/tensorflow/compiler/xla/xla_data.proto +++ b/tensorflow/compiler/xla/xla_data.proto @@ -48,6 +48,9 @@ enum PrimitiveType { F32 = 11; F64 = 12; + // Complex values of fixed width. + C64 = 15; // Paired F32 (real, imag), as in std::complex. + // A tuple is a polymorphic sequence; e.g. a shape that holds different // sub-shapes. They are used for things like returning multiple values from a // computation; e.g. a computation that returns weights and biases may have a @@ -305,6 +308,7 @@ message LiteralProto { repeated uint64 u64s = 7; repeated float f32s = 8; repeated double f64s = 9; + repeated float c64s = 12; // Stored as interleaved real, imag floats. repeated LiteralProto tuple_literals = 10; bytes f16s = 11; // Note: the F16s are encoded in little endian byte order } @@ -621,7 +625,7 @@ message WhileRequest { enum UnaryOperation { UNOP_INVALID = 0; - // Elementwise, logical negation + // Elementwise, logical negation on booleans and bitwise negation on ints. UNOP_NOT = 1; // Elementwise, computes e^x. @@ -663,6 +667,12 @@ enum UnaryOperation { // Elementwise, rounds x to nearest integral value, rounding half-way cases // away from zero. UNOP_ROUND_NEAREST_AFZ = 14; + + // Elementwise, extract real component of complex x. + UNOP_REAL = 15; + + // Elementwise, extract real component of complex x. + UNOP_IMAG = 16; } message UnaryOpRequest { @@ -690,14 +700,6 @@ enum BinaryOperation { // Dot product, matrix multiply. BINOP_DOT = 12; - // Indexes into the LHS with the RHS. - // - // If the RHS is higher-rank, this is a gather operation. - // - // Note: currently out of bounds indices may crash the underlying XLA - // machine. - BINOP_INDEX = 13; - // Element-wise maximum. BINOP_MAX = 14; @@ -710,13 +712,19 @@ enum BinaryOperation { // Remainder operation. BINOP_REM = 17; - // Logical operators + // Element-wise, logical operators on booleans and bitwise operators on ints. BINOP_AND = 18; BINOP_OR = 19; BINOP_SHIFT_LEFT = 20; BINOP_SHIFT_RIGHT_ARITHMETIC = 21; BINOP_SHIFT_RIGHT_LOGICAL = 22; + + // Complex from real, imag. + BINOP_COMPLEX = 23; + + // Computes the 4-quadrant arctangent of the y, x input arguments. + BINOP_ATAN2 = 24; } message BinaryOpRequest { @@ -755,10 +763,6 @@ enum TernaryOperation { // true and operand1 if the predicate is false. TRIOP_SELECT = 1; - // Updates operand0 at index operand1 with value operand2 and outputs the - // updated value. - TRIOP_UPDATE = 2; - // Given a min, max and an operand returns the operand if between min and max, // else returns min if operand is less than min or max if operand is greater // than max. @@ -800,18 +804,32 @@ message RecvRequest { ChannelHandle channel_handle = 2; } -message OpDeviceAssignment { - bool has_device = 1; - - // Number of the device to which this operator is assigned. Ignored if - // 'has_device' is false. - int32 device = 2; +message OpSharding { + enum Type { + // This sharding is replicated across all devices (implies maximal, + // all other fields are unused). + REPLICATED = 0; + // This sharding is maximal - one device runs the entire operation. + MAXIMAL = 1; + // Neither of the above; tile_shape and tile_assignment are both used. + OTHER = 2; + } + Type type = 1; + // The shape of the sharded tile. + Shape tile_shape = 2; + // The shape of the tile assignment tensor - this must be the same rank as + // tile_shape and the product of its dimensions must equal + // tile_assignment_devices.size(). + repeated int64 tile_assignment_dimensions = 3; + // Flattened list of device IDs. The order of flattening is the same as used + // by IndexUtil::MultiToLinearIndex(tile_assignment_shape). + repeated int64 tile_assignment_devices = 4; } message OpRequest { ComputationHandle computation = 1; OpMetadata metadata = 33; - OpDeviceAssignment device_assignment = 39; + OpSharding sharding = 40; oneof op { BinaryOpRequest binary_op_request = 2; @@ -850,7 +868,7 @@ message OpRequest { BatchNormTrainingRequest batch_norm_training_request = 35; BatchNormGradRequest batch_norm_grad_request = 37; BatchNormInferenceRequest batch_norm_inference_request = 38; - // Next: 40 + // Next: 41 } } diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD index 3d580fae142f990be249fb61119d23aa3c92210c..2e9b96bb1d31f7c985df992c094784660d6e274c 100644 --- a/tensorflow/contrib/BUILD +++ b/tensorflow/contrib/BUILD @@ -80,7 +80,7 @@ py_library( "//tensorflow/contrib/staging", "//tensorflow/contrib/stat_summarizer:stat_summarizer_py", "//tensorflow/contrib/stateless", - "//tensorflow/contrib/summary:summary_ops", + "//tensorflow/contrib/summary:summary", "//tensorflow/contrib/tensor_forest:init_py", "//tensorflow/contrib/tensorboard", "//tensorflow/contrib/testing:testing_py", @@ -88,8 +88,10 @@ py_library( "//tensorflow/contrib/tfprof", "//tensorflow/contrib/timeseries", "//tensorflow/contrib/tpu", + "//tensorflow/contrib/tpu:tpu_py", "//tensorflow/contrib/training:training_py", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:util", ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_ops_py"]), ) diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py index bf921808aa9a4694e06afcc2091b381a6fcffc49..a26fdb982c0f4d6d85b73912c194647a989d0ef6 100644 --- a/tensorflow/contrib/__init__.py +++ b/tensorflow/contrib/__init__.py @@ -77,9 +77,11 @@ from tensorflow.contrib import timeseries from tensorflow.contrib import tpu from tensorflow.contrib import training from tensorflow.contrib import util +from tensorflow.contrib.eager.python import tfe as eager from tensorflow.contrib.ndlstm import python as ndlstm from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph from tensorflow.contrib.specs import python as specs +from tensorflow.contrib.summary import summary from tensorflow.python.util.lazy_loader import LazyLoader ffmpeg = LazyLoader("ffmpeg", diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD index 744ae4c1f413bc1854a07ead9a3fa6bc90ed2fc1..8dff93b4f825277dcf0a64aa3b96bd809d36e1e9 100644 --- a/tensorflow/contrib/all_reduce/BUILD +++ b/tensorflow/contrib/all_reduce/BUILD @@ -19,9 +19,10 @@ py_library( srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - "//tensorflow/contrib/nccl:nccl_ops", + "//tensorflow/contrib/nccl:nccl_py", "//tensorflow/python:array_ops", "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", ], ) @@ -31,12 +32,17 @@ tf_py_test( additional_deps = [ ":all_reduce", "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:client", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:math_ops", "//tensorflow/python:constant_op", "//tensorflow/python:client_testlib", + "//tensorflow/python:platform", "//tensorflow/python:platform_test", + "//tensorflow/python:state_ops", ], ) diff --git a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java index 80e03f20362ed41b62ce118e864ffb0acb4ab50b..1f423a7a5bf6a115dc627ddd6f5e98c074282585 100644 --- a/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java +++ b/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java @@ -282,6 +282,22 @@ public class TensorFlowInferenceInterface { // Methods for taking a native Tensor and filling it with values from Java arrays. + /** + * Given a source array with shape {@link dims} and content {@link src}, copy the contents into + * the input Tensor with name {@link inputName}. The source array {@link src} must have at least + * as many elements as that of the destination Tensor. If {@link src} has more elements than the + * destination has capacity, the copy is truncated. + */ + public void feed(String inputName, boolean[] src, long... dims) { + byte[] b = new byte[src.length]; + + for (int i = 0; i < src.length; i++) { + b[i] = src[i] ? (byte) 1 : (byte) 0; + } + + addFeed(inputName, Tensor.create(Boolean.class, dims, ByteBuffer.wrap(b))); + } + /** * Given a source array with shape {@link dims} and content {@link src}, copy the contents into * the input Tensor with name {@link inputName}. The source array {@link src} must have at least diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD index 1555a3427fd5e40ca54c134a2c80f9d2c5feca36..8b7df4a84c558f662405a28a42426583d5ab39cd 100644 --- a/tensorflow/contrib/batching/BUILD +++ b/tensorflow/contrib/batching/BUILD @@ -69,6 +69,28 @@ tf_cc_test( ], ) +cc_library( + name = "adaptive_shared_batch_scheduler", + hdrs = ["adaptive_shared_batch_scheduler.h"], + deps = [ + ":batch_scheduler", + "//tensorflow/contrib/batching/util:periodic_function_dynamic", + "//tensorflow/core:lib", + ], +) + +tf_cc_test( + name = "adaptive_shared_batch_scheduler_test", + srcs = ["adaptive_shared_batch_scheduler_test.cc"], + deps = [ + ":adaptive_shared_batch_scheduler", + "//tensorflow/contrib/batching/test_util:fake_clock_env", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + cc_library( name = "basic_batch_scheduler", hdrs = ["basic_batch_scheduler.h"], @@ -155,14 +177,13 @@ tf_custom_op_py_library( deps = [ ":batch_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", + "//tensorflow/python:gradients", "//tensorflow/python:platform", - "//tensorflow/python:state_ops", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", + "//tensorflow/python:script_ops", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h new file mode 100644 index 0000000000000000000000000000000000000000..a0606427a526ffc67e10d12a084eabc64564e4ab --- /dev/null +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h @@ -0,0 +1,463 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ +#define THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ + +#include +#include +#include +#include +#include + +#include "tensorflow/contrib/batching/batch_scheduler.h" +#include "tensorflow/contrib/batching/util/periodic_function.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace serving { +namespace internal { +template +class ASBSBatch; + +template +class ASBSQueue; +} // namespace internal + +// Shared batch scheduler designed to minimize latency. The scheduler keeps +// track of a number of queues (one per model or model version) which are +// continuously enqueuing requests. The scheduler groups the requests into +// batches which it periodically sends off for processing (see +// shared_batch_scheduler.h for more details). The AdaptiveSharedBatchScheduler +// prioritizes batches by age (i.e. the batch's oldest request) irrespective of +// queue. The scheduler will process the oldest batch at an adjustable rate, +// regardless of batch size. The user can provide feedback to help set this rate +// to achieve some goal (i.e. minimize overall latency, limit cpu usage, etc). +// +// The rate (or rather, the corresponding period) is adjusted each time a batch +// is processed, using an exponentially weighted moving average to smooth +// potentially noisy feedback: +// ewma_feedback = ((N - 1) * ewma_feedback + feedback()) / N +// period *= (1 + K * emwa_feedback) +// +// Some potential use cases: +// Hardware Accelerators (GPUs & TPUs) - If some phase of batch processing +// involves serial processing by a device, from a latency perspective it is +// desirable to keep the device evenly loaded, avoiding the need to wait for +// the device to process prior batches. +// feedback = num_pending_on_device() - desired_pending. +// CPU utilization - If the batch processing is cpu dominated, you can reap +// latency gains when underutilized by increasing the processing rate, but +// back the rate off when the load increases to avoid overload. +// feedback = cpu_rate() - desired_cpu_rate. + +template +class AdaptiveSharedBatchScheduler + : public std::enable_shared_from_this< + AdaptiveSharedBatchScheduler> { + public: + struct Options { + // The name to use for the pool of batch threads. + string thread_pool_name = {"batch_threads"}; + // Number of batch processing threads; equivalently the maximum number of + // concurrently running batches. + int64 num_batch_threads = port::NumSchedulableCPUs(); + // The environment to use (typically only overridden by test code). + Env* env = Env::Default(); + // Initial batch scheduling period in microseconds. Will be altered for + // non-zero rate_feedback. + double initial_scheduling_period_micros = 500; + // Minimum batch scheduling period in microseconds. Recommend setting this + // value greater than 0, otherwise it may take a while to recover from a + // sustained time of negative scheduling_period_feedback (which may occur + // under low load). + double min_scheduling_period_micros = 100; + // Maximum batch scheduling period in microseconds. + double max_scheduling_period_micros = 10000; + // Feedback function used to modify the scheduling period each time a batch + // is scheduled. Should return values roughly O(1), with positive values + // resulting in an increased period. + std::function scheduling_period_feedback{[] { return 0.; }}; + // To handle potentially noisy scheduling_period_feedback, the period is + // adjusted using an exponentially weighted moving average over the previous + // feedback_smoothing_batches batches. Must be greater than 0. + int64 feedback_smoothing_batches = 10; + }; + + // Ownership is shared between the caller of Create() and any queues created + // via AddQueue(). + static Status Create( + const Options& options, + std::shared_ptr>* scheduler); + + struct QueueOptions { + // Maximum size of each batch. + int max_batch_size = 1000; + // Maximum number of enqueued (i.e. non-scheduled) batches. + int max_enqueued_batches = 10; + }; + + using BatchProcessor = std::function>)>; + + // Adds queue (and its callback) to be managed by this scheduler. + Status AddQueue(const QueueOptions& options, + BatchProcessor process_batch_callback, + std::unique_ptr>* queue); + + private: + // access to AddBatch, RemoveQueue, GetEnv. + friend class internal::ASBSQueue; + + explicit AdaptiveSharedBatchScheduler(const Options& options); + + // Batch scheduling function which runs every scheduling_period_ microseconds. + void ProcessOneBatch(); + + // Notifies scheduler of non-empty batch which is eligible for processing. + void AddBatch(internal::ASBSBatch*); + + // Removes queue from scheduler. + void RemoveQueue(const internal::ASBSQueue* queue); + + Env* GetEnv() const { return options_.env; } + + const Options options_; + + struct BatchCompare { + bool operator()(const internal::ASBSBatch* a, + const internal::ASBSBatch* b); + }; + + // Collection of batches added by AddBatch, ordered by age. Owned by scheduler + // until they are released for processing. + std::priority_queue*, + std::vector*>, BatchCompare> + batches_ GUARDED_BY(mu_); + + // Unowned queues and callbacks added by AddQueue. + std::unordered_map*, BatchProcessor> + queues_and_callbacks_ GUARDED_BY(mu_); + + mutex mu_; + + // Responsible for running ProcessOneBatch. PeriodicFunction was used in order + // to check for deletion so that the thread can be shut down. + std::unique_ptr scheduling_thread_; + + // Responsible for running the batch processing callbacks. + std::unique_ptr batch_thread_pool_; + + // Time interval in microseconds between successive ProcessOneBatch calls. + double scheduling_period_; + + // Exponentially weighted moving average of + // options_.scheduling_period_feedback() evaluated in each ProcessOneBatch + // call. + double ewma_feedback_ = 0; + + TF_DISALLOW_COPY_AND_ASSIGN(AdaptiveSharedBatchScheduler); +}; + +////////////////////////////////////////////////////////// +// Implementation details follow. API users need not read. + +namespace internal { +// Consolidates tasks into batches, passing them off to the +// AdaptiveSharedBatchScheduler for processing. +template +class ASBSQueue : public BatchScheduler { + public: + using QueueOptions = + typename AdaptiveSharedBatchScheduler::QueueOptions; + + ASBSQueue(std::shared_ptr> scheduler, + const QueueOptions& options); + + ~ASBSQueue() override; + + // Adds task to current batch. Fails if the task size is larger than the batch + // size or if the current batch is full and this queue's number of outstanding + // batches is at its maximum. + Status Schedule(std::unique_ptr* task) override; + + // Number of tasks waiting to be scheduled. + size_t NumEnqueuedTasks() const override; + + // Number of size 1 tasks which could currently be scheduled without failing. + size_t SchedulingCapacity() const override; + + // Notifies queue that a batch is about to be scheduled; the queue should not + // place any more tasks in this batch. + void ReleaseBatch(const ASBSBatch* batch); + + private: + std::shared_ptr> scheduler_; + const QueueOptions options_; + // Owned by scheduler_. + ASBSBatch* current_batch_ GUARDED_BY(mu_) = nullptr; + int64 num_enqueued_batches_ GUARDED_BY(mu_) = 0; + int64 num_enqueued_tasks_ GUARDED_BY(mu_) = 0; + mutable mutex mu_; + TF_DISALLOW_COPY_AND_ASSIGN(ASBSQueue); +}; + +// Batch which remembers when and by whom it was created. +template +class ASBSBatch : public Batch { + public: + ASBSBatch(ASBSQueue* queue, int64 creation_time_micros) + : queue_(queue), creation_time_micros_(creation_time_micros) {} + + ~ASBSBatch() override {} + + ASBSQueue* queue() const { return queue_; } + + int64 creation_time_micros() const { return creation_time_micros_; } + + private: + ASBSQueue* queue_; + const int64 creation_time_micros_; + TF_DISALLOW_COPY_AND_ASSIGN(ASBSBatch); +}; +} // namespace internal + +// ---------------- AdaptiveSharedBatchScheduler ---------------- + +template +Status AdaptiveSharedBatchScheduler::Create( + const Options& options, + std::shared_ptr>* scheduler) { + if (options.num_batch_threads < 1) { + return errors::InvalidArgument("num_batch_threads must be positive; was ", + options.num_batch_threads); + } + if (options.min_scheduling_period_micros < 0) { + return errors::InvalidArgument( + "min_scheduling_period_micros must be >= 0; was ", + options.min_scheduling_period_micros); + } + if (options.min_scheduling_period_micros > + options.initial_scheduling_period_micros) { + return errors::InvalidArgument( + "initial_scheduling_period_micros (", + options.initial_scheduling_period_micros, + ") must be >= min_scheduling_period_micros (", + options.min_scheduling_period_micros, ")"); + } + if (options.initial_scheduling_period_micros > + options.max_scheduling_period_micros) { + return errors::InvalidArgument( + "initial_scheduling_period_micros (", + options.initial_scheduling_period_micros, + ") must be <= max_scheduling_period_micros (", + options.max_scheduling_period_micros, ")"); + } + if (options.feedback_smoothing_batches < 1) { + return errors::InvalidArgument( + "feedback_smoothing_batches must be positive; was ", + options.feedback_smoothing_batches); + } + scheduler->reset(new AdaptiveSharedBatchScheduler(options)); + return Status::OK(); +} + +template +AdaptiveSharedBatchScheduler::AdaptiveSharedBatchScheduler( + const Options& options) + : options_(options), + scheduling_period_(options.initial_scheduling_period_micros) { + PeriodicFunction::Options opts; + opts.thread_name_prefix = "scheduling_thread"; + opts.env = GetEnv(); + scheduling_thread_.reset( + new PeriodicFunction([this] { ProcessOneBatch(); }, 0, opts)); + batch_thread_pool_.reset(new thread::ThreadPool( + GetEnv(), options.thread_pool_name, options.num_batch_threads)); +} + +template +Status AdaptiveSharedBatchScheduler::AddQueue( + const QueueOptions& options, BatchProcessor process_batch_callback, + std::unique_ptr>* queue) { + if (options.max_batch_size <= 0) { + return errors::InvalidArgument("max_batch_size must be positive; was ", + options.max_batch_size); + } + if (options.max_enqueued_batches <= 0) { + return errors::InvalidArgument( + "max_enqueued_batches must be positive; was ", + options.max_enqueued_batches); + } + internal::ASBSQueue* asbs_queue_raw; + queue->reset(asbs_queue_raw = new internal::ASBSQueue( + this->shared_from_this(), options)); + mutex_lock l(mu_); + queues_and_callbacks_[asbs_queue_raw] = process_batch_callback; + return Status::OK(); +} + +template +void AdaptiveSharedBatchScheduler::AddBatch( + internal::ASBSBatch* batch) { + mutex_lock l(mu_); + batches_.push(batch); +} + +template +void AdaptiveSharedBatchScheduler::RemoveQueue( + const internal::ASBSQueue* queue) { + mutex_lock l(mu_); + queues_and_callbacks_.erase(queue); +} + +template +void AdaptiveSharedBatchScheduler::ProcessOneBatch() { + static const double kFeedbackMultiplier = .001; + internal::ASBSBatch* batch = nullptr; + BatchProcessor callback; + const int64 start_time_micros = GetEnv()->NowMicros(); + { + mutex_lock l(mu_); + if (!batches_.empty()) { + batch = batches_.top(); + batches_.pop(); + callback = queues_and_callbacks_[batch->queue()]; + } + } + if (batch != nullptr) { + double feedback = options_.scheduling_period_feedback(); + const int64 N = options_.feedback_smoothing_batches; + ewma_feedback_ = ((N - 1) * ewma_feedback_ + feedback) / N; + scheduling_period_ *= (1 + kFeedbackMultiplier * ewma_feedback_); + if (scheduling_period_ < options_.min_scheduling_period_micros) { + scheduling_period_ = options_.min_scheduling_period_micros; + } else if (scheduling_period_ > options_.max_scheduling_period_micros) { + scheduling_period_ = options_.max_scheduling_period_micros; + } + // Queue may destroy itself after ReleaseBatch is called. + batch->queue()->ReleaseBatch(batch); + batch_thread_pool_->Schedule([callback, batch] { + callback(std::unique_ptr>(batch)); + }); + } + const int64 sleep_time = + scheduling_period_ - (GetEnv()->NowMicros() - start_time_micros); + if (sleep_time > 0) { + GetEnv()->SleepForMicroseconds(sleep_time); + } +} + +template +bool AdaptiveSharedBatchScheduler::BatchCompare::operator()( + const internal::ASBSBatch* a, + const internal::ASBSBatch* b) { + return a->creation_time_micros() > b->creation_time_micros(); +} + +// ---------------- ASBSQueue ---------------- + +namespace internal { +template +ASBSQueue::ASBSQueue( + std::shared_ptr> scheduler, + const QueueOptions& options) + : scheduler_(scheduler), options_(options) {} + +template +ASBSQueue::~ASBSQueue() { + // Wait until last batch has been scheduled. + const int kSleepMicros = 1000; + for (;;) { + { + mutex_lock l(mu_); + if (num_enqueued_batches_ == 0) { + break; + } + } + scheduler_->GetEnv()->SleepForMicroseconds(kSleepMicros); + } + scheduler_->RemoveQueue(this); +} + +template +Status ASBSQueue::Schedule(std::unique_ptr* task) { + bool added_new_batch = false; + size_t size = (*task)->size(); + if (size > options_.max_batch_size) { + return errors::InvalidArgument("Task size ", size, + " is larger than maximum batch size ", + options_.max_batch_size); + } + { + mutex_lock l(mu_); + // Current batch is full, create another if allowed. + if (current_batch_ && + current_batch_->size() + size > options_.max_batch_size) { + if (num_enqueued_batches_ >= options_.max_enqueued_batches) { + return errors::Unavailable("The batch scheduling queue is full"); + } + current_batch_->Close(); + current_batch_ = nullptr; + } + if (!current_batch_) { + added_new_batch = true; + num_enqueued_batches_++; + current_batch_ = + new ASBSBatch(this, scheduler_->GetEnv()->NowMicros()); + } + current_batch_->AddTask(std::move(*task)); + num_enqueued_tasks_++; + } + if (added_new_batch) scheduler_->AddBatch(current_batch_); + return Status::OK(); +} + +template +void ASBSQueue::ReleaseBatch(const ASBSBatch* batch) { + mutex_lock l(mu_); + num_enqueued_batches_--; + num_enqueued_tasks_ -= batch->num_tasks(); + if (batch == current_batch_) { + current_batch_->Close(); + current_batch_ = nullptr; + } +} + +template +size_t ASBSQueue::NumEnqueuedTasks() const { + mutex_lock l(mu_); + return num_enqueued_tasks_; +} + +template +size_t ASBSQueue::SchedulingCapacity() const { + mutex_lock l(mu_); + const int current_batch_capacity = + current_batch_ ? options_.max_batch_size - current_batch_->size() : 0; + const int spare_batches = + options_.max_enqueued_batches - num_enqueued_batches_; + return spare_batches * options_.max_batch_size + current_batch_capacity; +} +} // namespace internal +} // namespace serving +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CONTRIB_BATCHING_ADAPTIVE_SHARED_BATCH_SCHEDULER_H_ diff --git a/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..a07cd6d834fa28904bf7748b16972cca217503c1 --- /dev/null +++ b/tensorflow/contrib/batching/adaptive_shared_batch_scheduler_test.cc @@ -0,0 +1,438 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/contrib/batching/adaptive_shared_batch_scheduler.h" + +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace serving { +namespace anonymous { + +class FakeTask : public BatchTask { + public: + explicit FakeTask(size_t size) : size_(size) {} + + ~FakeTask() override = default; + + size_t size() const override { return size_; } + + private: + const size_t size_; + + TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); +}; + +// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on +// that task. Returns the resulting status. +Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { + std::unique_ptr task(new FakeTask(task_size)); + Status status = scheduler->Schedule(&task); + // Schedule() should have consumed 'task' iff it returned Status::OK. + CHECK_EQ(status.ok(), task == nullptr); + return status; +} + +// Creates a thread that waits on 'start' and then advances the fake clock in +// 'env' in a loop until 'stop' is notified. Useful for allowing objects that +// use the clock to be destroyed. +std::unique_ptr CreateFakeClockAdvancerThread( + test_util::FakeClockEnv* env, Notification* start, Notification* stop) { + return std::unique_ptr(Env::Default()->StartThread( + {}, "FakeClockAdvancerThread", [env, start, stop] { + start->WaitForNotification(); + while (!stop->HasBeenNotified()) { + env->AdvanceByMicroseconds(10); + Env::Default()->SleepForMicroseconds(10); + } + })); +} + +TEST(AdaptiveSharedBatchSchedulerTest, Basic) { + for (const bool delete_scheduler_early : {false, true}) { + for (const bool delete_queue_1_early : {false, true}) { + int queue_0_tasks = 0; + auto queue_0_callback = + [&queue_0_tasks](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_0_tasks += batch->task(i).size(); + } + }; + int queue_1_tasks = 0; + auto queue_1_callback = + [&queue_1_tasks](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_1_tasks += batch->task(i).size(); + } + }; + { + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create({}, &scheduler)); + + // Create two queues. + std::unique_ptr> queue_0; + TF_ASSERT_OK(scheduler->AddQueue({}, queue_0_callback, &queue_0)); + std::unique_ptr> queue_1; + TF_ASSERT_OK(scheduler->AddQueue({}, queue_1_callback, &queue_1)); + + if (delete_scheduler_early) { + // Delete our copy of the scheduler. The queues should keep it alive + // under the covers. + scheduler = nullptr; + } + // Submit tasks to the two queues, and (optionally) remove the queues. + TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); + TF_ASSERT_OK(ScheduleTask(3, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(4, queue_1.get())); + if (delete_queue_1_early) { + queue_1 = nullptr; + } + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + } + EXPECT_EQ(queue_0_tasks, 9); + EXPECT_EQ(queue_1_tasks, 6); + } + } +} + +TEST(AdaptiveSharedBatchSchedulerTest, BadOptions) { + using Scheduler = AdaptiveSharedBatchScheduler; + std::shared_ptr scheduler; + Scheduler::Options options; + options.num_batch_threads = 0; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 50; + options.max_scheduling_period_micros = 100; + options.initial_scheduling_period_micros = 1; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 50; + options.max_scheduling_period_micros = 100; + options.initial_scheduling_period_micros = 1000; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.min_scheduling_period_micros = 100; + options.max_scheduling_period_micros = 50; + options.initial_scheduling_period_micros = 75; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); + options = Scheduler::Options(); + options.feedback_smoothing_batches = 0; + EXPECT_FALSE(Scheduler::Create(options, &scheduler).ok()); +} + +TEST(AdaptiveSharedBatchSchedulerTest, ObeysQueueOptions) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue_0; + std::unique_ptr> queue_1; + int queue_0_tasks = 0; + int queue_1_tasks = 0; + auto queue_0_callback = [&queue_0_tasks, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_0_tasks += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + auto queue_1_callback = [&queue_1_tasks, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + for (int i = 0; i < batch->num_tasks(); i++) { + queue_1_tasks += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + AdaptiveSharedBatchScheduler::QueueOptions queue_options; + queue_options.max_batch_size = 10; + queue_options.max_enqueued_batches = 0; + // Queue must have max_enqueued_batchs > 1. + EXPECT_FALSE( + scheduler->AddQueue(queue_options, queue_0_callback, &queue_0).ok()); + queue_options.max_enqueued_batches = 2; + TF_ASSERT_OK( + scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); + queue_options.max_batch_size = 0; + // Queue must have max_batch_size > 0. + EXPECT_FALSE( + scheduler->AddQueue(queue_options, queue_1_callback, &queue_1).ok()); + queue_options.max_batch_size = 2; + queue_options.max_enqueued_batches = 1; + TF_ASSERT_OK( + scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Task larger than max_batch_size shouldn't schedule. + EXPECT_FALSE(ScheduleTask(15, queue_0.get()).ok()); + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + env.AdvanceByMicroseconds(1); + + // Task larger than max_batch_size shouldn't schedule. + EXPECT_FALSE(ScheduleTask(3, queue_1.get()).ok()); + TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); + TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); + env.AdvanceByMicroseconds(1); + // Exceeds max_enqueued_batches, shouldn't schedule. + EXPECT_FALSE(ScheduleTask(1, queue_1.get()).ok()); + + TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); + // Exceeds max_enqueued_batches, shouldn't schedule. + EXPECT_FALSE(ScheduleTask(6, queue_0.get()).ok()); + TF_ASSERT_OK(ScheduleTask(4, queue_0.get())); + + // Batches should be processed in order from oldest to newest. + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 10); + EXPECT_EQ(queue_1_tasks, 0); + + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 10); + EXPECT_EQ(queue_1_tasks, 2); + + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(queue_0_tasks, 19); + EXPECT_EQ(queue_1_tasks, 2); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, RateFeedback) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + double feedback = 0; + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.min_scheduling_period_micros = 200; + options.max_scheduling_period_micros = 2000; + options.env = &env; + options.scheduling_period_feedback = [&feedback] { return feedback; }; + options.feedback_smoothing_batches = 1; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + + TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 6 batches. + for (int i = 0; i < 6; i++) { + TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); + env.AdvanceByMicroseconds(1); + } + feedback = -500; + env.AdvanceByMicroseconds(994); + env.BlockUntilThreadsAsleep(2); // scheduling period = 500 usec. + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(500); + env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. + EXPECT_EQ(scheduled_items, 901); + feedback = 0; + env.AdvanceByMicroseconds(250); + env.BlockUntilThreadsAsleep(2); // scheduling period = 250 usec. + EXPECT_EQ(scheduled_items, 902); + feedback = 10000; // large feedback should hit max_scheduling_period. + env.AdvanceByMicroseconds(250); + env.BlockUntilThreadsAsleep(2); // scheduling period = 2000 usec. + EXPECT_EQ(scheduled_items, 903); + feedback = -10000; // large feedback should hit min_scheduling_period. + env.AdvanceByMicroseconds(1999); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 903); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); // scheduling period = 200 usec. + EXPECT_EQ(scheduled_items, 904); + env.AdvanceByMicroseconds(200); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 905); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, FeedbackSmoothing) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + double feedback = 0; + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + options.scheduling_period_feedback = [&feedback] { return feedback; }; + options.feedback_smoothing_batches = 3; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + + TF_ASSERT_OK(scheduler->AddQueue({}, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 4 batches. + for (int i = 0; i < 4; i++) { + TF_ASSERT_OK(ScheduleTask(900 + i, queue.get())); + env.AdvanceByMicroseconds(1); + } + feedback = -300; + env.AdvanceByMicroseconds(996); + env.BlockUntilThreadsAsleep(2); + // ewma_feedback = 100, scheduling_period = 900. + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(899); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 900); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + // ewma_feedback = 167, scheduling_period = 750. + EXPECT_EQ(scheduled_items, 901); + env.AdvanceByMicroseconds(749); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 901); + feedback = 1000 / 3.; + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + // emwa_feedback = 0, scheduling_period = 750. + EXPECT_EQ(scheduled_items, 902); + env.AdvanceByMicroseconds(749); + // No callback scheduled, only scheduling thread sleeping. + env.BlockUntilThreadsAsleep(1); + EXPECT_EQ(scheduled_items, 902); + env.AdvanceByMicroseconds(1); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 903); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} + +TEST(AdaptiveSharedBatchSchedulerTest, QueueCapacityInfo) { + test_util::FakeClockEnv env(Env::Default()); + Notification start_teardown, stop_teardown; + std::unique_ptr teardown_thread = + CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); + { + AdaptiveSharedBatchScheduler::Options options; + options.initial_scheduling_period_micros = 1000; + options.env = &env; + std::shared_ptr> scheduler; + TF_ASSERT_OK( + AdaptiveSharedBatchScheduler::Create(options, &scheduler)); + std::unique_ptr> queue; + int scheduled_items = 0; + auto queue_callback = [&scheduled_items, + &env](std::unique_ptr> batch) { + ASSERT_TRUE(batch->IsClosed()); + EXPECT_GT(batch->num_tasks(), 0); + scheduled_items = 0; + for (int i = 0; i < batch->num_tasks(); i++) { + scheduled_items += batch->task(i).size(); + } + env.SleepForMicroseconds(1); + }; + AdaptiveSharedBatchScheduler::QueueOptions queue_options; + queue_options.max_batch_size = 10; + queue_options.max_enqueued_batches = 10; + TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_callback, &queue)); + + // Wait for scheduling_thread to sleep. + env.BlockUntilThreadsAsleep(1); + // Enqueue 3 tasks. + EXPECT_EQ(queue->NumEnqueuedTasks(), 0); + EXPECT_EQ(queue->SchedulingCapacity(), 100); + TF_ASSERT_OK(ScheduleTask(5, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 1); + EXPECT_EQ(queue->SchedulingCapacity(), 95); + env.AdvanceByMicroseconds(1); + TF_ASSERT_OK(ScheduleTask(6, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 2); + EXPECT_EQ(queue->SchedulingCapacity(), 84); + env.AdvanceByMicroseconds(1); + TF_ASSERT_OK(ScheduleTask(1, queue.get())); + EXPECT_EQ(queue->NumEnqueuedTasks(), 3); + EXPECT_EQ(queue->SchedulingCapacity(), 83); + + env.AdvanceByMicroseconds(998); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 5); + env.AdvanceByMicroseconds(1000); + env.BlockUntilThreadsAsleep(2); + EXPECT_EQ(scheduled_items, 7); + start_teardown.Notify(); + } + stop_teardown.Notify(); +} +} // namespace anonymous +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow/contrib/batching/batch_scheduler.h b/tensorflow/contrib/batching/batch_scheduler.h index 7c41ad88180badd37398f5bae057dcd0006922c3..a5072f439abad3c5db79a514a7f2baff0b021b39 100644 --- a/tensorflow/contrib/batching/batch_scheduler.h +++ b/tensorflow/contrib/batching/batch_scheduler.h @@ -78,7 +78,7 @@ template class Batch { public: Batch() = default; - ~Batch(); // Blocks until the batch is closed. + virtual ~Batch(); // Blocks until the batch is closed. // Appends 'task' to the batch. After calling AddTask(), the newly-added task // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1). diff --git a/tensorflow/contrib/bayesflow/BUILD b/tensorflow/contrib/bayesflow/BUILD index 324e519a6dbfac859d386576578f7989db0cc3c5..8bb742d289a0836378a9a03c90d46293cfbfe75b 100644 --- a/tensorflow/contrib/bayesflow/BUILD +++ b/tensorflow/contrib/bayesflow/BUILD @@ -20,8 +20,9 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:functional_ops", + "//tensorflow/python:gradients", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", @@ -31,7 +32,6 @@ py_library( "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", "//tensorflow/python/ops/distributions", "//third_party/py/numpy", "@six_archive//:six", diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD index 1b85c260c0ce6a4a7e772b07aa5d639105232f5f..66a04d42e93331de74b6f3d41f83f071115c1097 100644 --- a/tensorflow/contrib/boosted_trees/BUILD +++ b/tensorflow/contrib/boosted_trees/BUILD @@ -81,9 +81,6 @@ py_test( size = "small", srcs = ["python/utils/losses_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":losses", "//tensorflow/python:array_ops", @@ -135,7 +132,6 @@ py_test( srcs = ["python/training/functions/gbdt_batch_test.py"], srcs_version = "PY2AND3", tags = [ - "nomac", # b/63258195 "notsan", # b/62863147 ], deps = [ @@ -164,9 +160,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/model_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":model_ops_py", ":prediction_ops_py", @@ -187,9 +180,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/prediction_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":model_ops_py", ":prediction_ops_py", @@ -207,9 +197,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/quantile_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":quantile_ops_py", "//tensorflow/contrib/boosted_trees/proto:quantiles_proto_py", @@ -247,9 +234,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/stats_accumulator_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":stats_accumulator_ops_py", "//tensorflow/python:framework_ops", @@ -264,9 +248,6 @@ py_test( size = "small", srcs = ["python/kernel_tests/training_ops_test.py"], srcs_version = "PY2AND3", - tags = [ - "nomac", # b/63258195 - ], deps = [ ":model_ops_py", ":training_ops_py", diff --git a/tensorflow/contrib/boosted_trees/README.md b/tensorflow/contrib/boosted_trees/README.md index 9ce700f1a19fde3f5b07748fd6768e9e8e336c8a..7d30032e539fb16e27f48ea101094fa4d3e9171d 100644 --- a/tensorflow/contrib/boosted_trees/README.md +++ b/tensorflow/contrib/boosted_trees/README.md @@ -1,7 +1,7 @@ # TF Boosted Trees (TFBT) TF Boosted trees is an implementation of a gradient boosting algorithm with -trees used as week learners. +trees used as weak learners. ## Examples Folder "examples" demonstrates how TFBT estimators can be used for various diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD index d0ee1fd60d0b62395f6638ab3d67e6fe95ae8331..7792c7127c0285dc2eb5b213da054674f6a81d64 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD +++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD @@ -124,6 +124,8 @@ py_library( srcs_version = "PY2AND3", deps = [ ":model", + "//tensorflow/contrib/boosted_trees:losses", "//tensorflow/contrib/learn", + "//tensorflow/python:math_ops", ], ) diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py index a800c3ddc7954133652d53b8fa381d4f1b3b5d40..ef8dee91b6cc05c4c3dd5eb3c81de4fb65b473e3 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py @@ -149,6 +149,8 @@ def convert_to_universal_format(dtec, sorted_feature_names, split = gtflow_node.sparse_float_binary_split_default_left.split node.default_direction = ( generic_tree_model_pb2.BinaryNode.LEFT) + # TODO(nponomareva): adjust this id assignement when we allow multi- + # column sparse tensors. feature_id = split.feature_column + num_dense inequality_test = node.inequality_left_child_test inequality_test.feature_id.id.value = sorted_feature_names[feature_id] @@ -159,6 +161,8 @@ def convert_to_universal_format(dtec, sorted_feature_names, split = gtflow_node.sparse_float_binary_split_default_right.split node.default_direction = ( generic_tree_model_pb2.BinaryNode.RIGHT) + # TODO(nponomareva): adjust this id assignement when we allow multi- + # column sparse tensors. feature_id = split.feature_column + num_dense inequality_test = node.inequality_left_child_test inequality_test.feature_id.id.value = sorted_feature_names[feature_id] diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py index f8028acbdb0be44b7fd81b96b04b6e24d9060aa6..01752416b347dd0a5e646283b6b5572592df4690 100644 --- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py +++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py @@ -19,8 +19,10 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.boosted_trees.estimator_batch import model +from tensorflow.contrib.boosted_trees.python.utils import losses from tensorflow.contrib.learn.python.learn.estimators import estimator from tensorflow.contrib.learn.python.learn.estimators import head as head_lib +from tensorflow.python.ops import math_ops class GradientBoostedDecisionTreeClassifier(estimator.Estimator): @@ -65,10 +67,21 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator): Raises: ValueError: If learner_config is not valid. """ + if n_classes > 2: + # For multi-class classification, use our loss implementation that + # supports second order derivative. + def loss_fn(labels, logits, weights=None): + result = losses.per_example_maxent_loss( + labels=labels, logits=logits, weights=weights, + num_classes=n_classes) + return math_ops.reduce_mean(result[0]) + else: + loss_fn = None head = head_lib.multi_class_head( n_classes=n_classes, weight_column_name=weight_column_name, - enable_centered_bias=False) + enable_centered_bias=False, + loss_fn=loss_fn) if learner_config.num_classes == 0: learner_config.num_classes = n_classes elif learner_config.num_classes != n_classes: diff --git a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py index 9be362f5c834c3f4e5ae37711865e6364c1ec5e4..47ee3d816f41e44f3a2458cf537d4f7dccf7b614 100644 --- a/tensorflow/contrib/boosted_trees/examples/binary_mnist.py +++ b/tensorflow/contrib/boosted_trees/examples/binary_mnist.py @@ -21,7 +21,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example. python tensorflow/contrib/boosted_trees/examples/binary_mnist.py \ --output_dir="/tmp/binary_mnist" --depth=4 --learning_rate=0.3 \ --batch_size=10761 --examples_per_layer=10761 --eval_batch_size=1030 \ - --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 \ + --num_eval_steps=1 --num_trees=10 --l2=1 --vmodule=training_ops=1 When training is done, accuracy on eval data is reported. Point tensorboard to the directory for the run to see how the training progresses: @@ -52,7 +52,7 @@ def get_input_fn(data, ids = np.where((data.labels == 4) | (data.labels == 9)) images = data.images[ids] labels = data.labels[ids] - # Make digit 4 label 0, 9 is 1. + # Make digit 4 label 1, 9 is 0. labels = labels == 4 def _input_fn(): diff --git a/tensorflow/contrib/boosted_trees/examples/mnist.py b/tensorflow/contrib/boosted_trees/examples/mnist.py index a3b1cb5154644e1a97633d429aae8ae18ecdaa2b..817c6eb3e1a79b38746418db9e5015e65ee70a50 100644 --- a/tensorflow/contrib/boosted_trees/examples/mnist.py +++ b/tensorflow/contrib/boosted_trees/examples/mnist.py @@ -22,7 +22,7 @@ r"""Demonstrates multiclass MNIST TF Boosted trees example. python tensorflow/contrib/boosted_trees/examples/mnist.py \ --output_dir="/tmp/mnist" --depth=4 --learning_rate=0.3 --batch_size=60000 \ --examples_per_layer=60000 --eval_batch_size=10000 --num_eval_steps=1 \ - --num_trees=10 --l2=1 --vmodule=training_ops=1 \ + --num_trees=10 --l2=1 --vmodule=training_ops=1 When training is done, accuracy on eval data is reported. Point tensorboard to the directory for the run to see how the training progresses: @@ -35,18 +35,13 @@ from __future__ import division from __future__ import print_function import argparse -import functools import sys import numpy as np import tensorflow as tf -from tensorflow.contrib import metrics as metrics_lib -from tensorflow.contrib.boosted_trees.estimator_batch import custom_loss_head -from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeEstimator +from tensorflow.contrib.boosted_trees.estimator_batch.estimator import GradientBoostedDecisionTreeClassifier from tensorflow.contrib.boosted_trees.proto import learner_pb2 -from tensorflow.contrib.boosted_trees.python.utils import losses from tensorflow.contrib.learn import learn_runner -from tensorflow.python.ops import math_ops def get_input_fn(dataset_split, @@ -88,36 +83,13 @@ def _get_tfbt(output_dir): learner_config.growing_mode = growing_mode run_config = tf.contrib.learn.RunConfig(save_checkpoints_secs=300) - # Use Cross Entropy loss (the impl in losses is twice differentiable). - loss_fn = functools.partial( - losses.per_example_maxent_loss, num_classes=num_classes) - logit_dim = num_classes learner_config.multi_class_strategy = ( learner_pb2.LearnerConfig.DIAGONAL_HESSIAN) - # Since we use custom head, we need to tell how accuracy is calculated. - def _multiclass_metrics(predictions, labels, weights): - """Prepares eval metrics for multiclass eval.""" - metrics = dict() - logits = predictions["scores"] - classes = math_ops.argmax(logits, 1) - metrics["accuracy"] = metrics_lib.streaming_accuracy( - classes, labels, weights) - return metrics - - metrics_fn = _multiclass_metrics - # Use custom loss head so we can provide our loss (cross entropy for - # multiclass). - head = custom_loss_head.CustomLossHead( - loss_fn=loss_fn, - link_fn=tf.identity, - logit_dimension=logit_dim, - metrics_fn=metrics_fn) - # Create a TF Boosted trees estimator that can take in custom loss. - estimator = GradientBoostedDecisionTreeEstimator( + estimator = GradientBoostedDecisionTreeClassifier( learner_config=learner_config, - head=head, + n_classes=num_classes, examples_per_layer=FLAGS.examples_per_layer, model_dir=output_dir, num_trees=FLAGS.num_trees, diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD index 70aa0284a6fcd822b854888259b41cdf60d22af5..107ff0d295bee530c1711a97849fbd3c6cdb2f00 100644 --- a/tensorflow/contrib/boosted_trees/lib/BUILD +++ b/tensorflow/contrib/boosted_trees/lib/BUILD @@ -81,6 +81,18 @@ tf_cc_test( ], ) +tf_cc_test( + name = "example_test", + size = "small", + srcs = ["utils/example_test.cc"], + deps = [ + ":utils", + "//tensorflow/core:tensor_testutil", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + tf_cc_test( name = "batch_features_test", size = "small", diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc index bd70586393eb062a46b6e242c6094ef0605804e2..f8750e7191673274772fc869c198dd5fbbefbc49 100644 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc +++ b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree.cc @@ -50,10 +50,15 @@ int DecisionTree::Traverse(const DecisionTreeConfig& config, current_node.sparse_float_binary_split_default_left().split(); auto sparse_feature = example.sparse_float_features[split.feature_column()]; - node_id = !sparse_feature.has_value() || - sparse_feature.get_value() <= split.threshold() - ? split.left_id() - : split.right_id(); + // Feature id for the split when multivalent sparse float column, or 0 + // by default. + const int32 feature_id = split.feature_id(); + + node_id = + !sparse_feature[feature_id].has_value() || + sparse_feature[feature_id].get_value() <= split.threshold() + ? split.left_id() + : split.right_id(); break; } case TreeNode::kSparseFloatBinarySplitDefaultRight: { @@ -61,10 +66,14 @@ int DecisionTree::Traverse(const DecisionTreeConfig& config, current_node.sparse_float_binary_split_default_right().split(); auto sparse_feature = example.sparse_float_features[split.feature_column()]; - node_id = sparse_feature.has_value() && - sparse_feature.get_value() <= split.threshold() - ? split.left_id() - : split.right_id(); + // Feature id for the split when multivalent sparse float column, or 0 + // by default. + const int32 feature_id = split.feature_id(); + node_id = + sparse_feature[feature_id].has_value() && + sparse_feature[feature_id].get_value() <= split.threshold() + ? split.left_id() + : split.right_id(); break; } case TreeNode::kCategoricalIdBinarySplit: { diff --git a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc index c55d09807eaf3a9c9db1cfbbfdfc66aec8f25155..93924d429c19aef51b6f1d85655de3798a76e3e0 100644 --- a/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/trees/decision_tree_test.cc @@ -27,13 +27,14 @@ class DecisionTreeTest : public ::testing::Test { protected: DecisionTreeTest() : batch_features_(2) { // Create a batch of two examples having one dense float, two sparse float - // and one sparse int features. + // and one sparse int features, and one sparse multi-column float feature + // (SparseFM). // The first example is missing the second sparse feature column and the // second example is missing the first sparse feature column. // This looks like the following: - // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | - // 0 | 7 | -3 | | 3 | - // 1 | -2 | | 4 | | + // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseFM (3 cols) + // 0 | 7 | -3 | | 3 | 3.0 | | 1.0 + // 1 | -2 | | 4 | | 1.5 |3.5| auto dense_float_matrix = test::AsTensor({7.0f, -2.0f}, {2, 1}); auto sparse_float_indices1 = test::AsTensor({0, 0}, {1, 2}); auto sparse_float_values1 = test::AsTensor({-3.0f}); @@ -44,11 +45,21 @@ class DecisionTreeTest : public ::testing::Test { auto sparse_int_indices1 = test::AsTensor({0, 0}, {1, 2}); auto sparse_int_values1 = test::AsTensor({3}); auto sparse_int_shape1 = test::AsTensor({2, 1}); + + // Multivalent sparse feature. + auto multi_sparse_float_indices = + test::AsTensor({0, 0, 0, 2, 1, 0, 1, 1}, {4, 2}); + auto multi_sparse_float_values = + test::AsTensor({3.0f, 1.0f, 1.5f, 3.5f}); + auto multi_sparse_float_shape = test::AsTensor({2, 3}); + TF_EXPECT_OK(batch_features_.Initialize( - {dense_float_matrix}, {sparse_float_indices1, sparse_float_indices2}, - {sparse_float_values1, sparse_float_values2}, - {sparse_float_shape1, sparse_float_shape2}, {sparse_int_indices1}, - {sparse_int_values1}, {sparse_int_shape1})); + {dense_float_matrix}, + {sparse_float_indices1, sparse_float_indices2, + multi_sparse_float_indices}, + {sparse_float_values1, sparse_float_values2, multi_sparse_float_values}, + {sparse_float_shape1, sparse_float_shape2, multi_sparse_float_shape}, + {sparse_int_indices1}, {sparse_int_values1}, {sparse_int_shape1})); } template @@ -121,44 +132,90 @@ TEST_F(DecisionTreeTest, TraverseDenseBinarySplit) { } TEST_F(DecisionTreeTest, TraverseSparseBinarySplit) { - // Test first sparse feature which is missing for the second example. - DecisionTreeConfig tree_config1; - auto* split_node1 = tree_config1.add_nodes() - ->mutable_sparse_float_binary_split_default_left() - ->mutable_split(); - split_node1->set_feature_column(0); - split_node1->set_threshold(-20.0f); - split_node1->set_left_id(1); - split_node1->set_right_id(2); - tree_config1.add_nodes()->mutable_leaf(); - tree_config1.add_nodes()->mutable_leaf(); auto example_iterable = batch_features_.examples_iterable(0, 2); - - // Expect right child to be picked as !(-3 <= -20). - auto example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config1, 0, *example_it)); - - // Expect left child to be picked as default direction. - EXPECT_EQ(1, DecisionTree::Traverse(tree_config1, 0, *++example_it)); - + // Split on SparseF1. + // Test first sparse feature which is missing for the second example. + { + DecisionTreeConfig tree_config; + auto* split_node = tree_config.add_nodes() + ->mutable_sparse_float_binary_split_default_left() + ->mutable_split(); + split_node->set_feature_column(0); + split_node->set_threshold(-20.0f); + split_node->set_left_id(1); + split_node->set_right_id(2); + tree_config.add_nodes()->mutable_leaf(); + tree_config.add_nodes()->mutable_leaf(); + + // Expect right child to be picked as !(-3 <= -20). + auto example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + + // Expect left child to be picked as default direction. + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + } + // Split on SparseF2. // Test second sparse feature which is missing for the first example. - DecisionTreeConfig tree_config2; - auto* split_node2 = tree_config2.add_nodes() - ->mutable_sparse_float_binary_split_default_right() - ->mutable_split(); - split_node2->set_feature_column(1); - split_node2->set_threshold(4.0f); - split_node2->set_left_id(1); - split_node2->set_right_id(2); - tree_config2.add_nodes()->mutable_leaf(); - tree_config2.add_nodes()->mutable_leaf(); - - // Expect right child to be picked as default direction. - example_it = example_iterable.begin(); - EXPECT_EQ(2, DecisionTree::Traverse(tree_config2, 0, *example_it)); - - // Expect left child to be picked as (4 <= 4). - EXPECT_EQ(1, DecisionTree::Traverse(tree_config2, 0, *++example_it)); + { + DecisionTreeConfig tree_config; + auto* split_node = tree_config.add_nodes() + ->mutable_sparse_float_binary_split_default_right() + ->mutable_split(); + split_node->set_feature_column(1); + split_node->set_threshold(4.0f); + split_node->set_left_id(1); + split_node->set_right_id(2); + tree_config.add_nodes()->mutable_leaf(); + tree_config.add_nodes()->mutable_leaf(); + + // Expect right child to be picked as default direction. + auto example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + + // Expect left child to be picked as (4 <= 4). + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + } + // Split on SparseFM. + // Test second sparse feature which is missing for the first example. + { + DecisionTreeConfig tree_config; + auto* split_node = tree_config.add_nodes() + ->mutable_sparse_float_binary_split_default_right() + ->mutable_split(); + split_node->set_feature_column(2); + + split_node->set_left_id(1); + split_node->set_right_id(2); + tree_config.add_nodes()->mutable_leaf(); + tree_config.add_nodes()->mutable_leaf(); + + // Split on first column + split_node->set_feature_id(0); + split_node->set_threshold(2.0f); + + // Both instances have this feature value. + auto example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + + // Split on second column + split_node->set_feature_id(1); + split_node->set_threshold(5.0f); + + // First instance does not have it (default right), second does have it. + example_it = example_iterable.begin(); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *example_it)); + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *++example_it)); + + // Split on third column + split_node->set_feature_id(2); + split_node->set_threshold(3.0f); + example_it = example_iterable.begin(); + + // First instance has it, second does not (default right). + EXPECT_EQ(1, DecisionTree::Traverse(tree_config, 0, *example_it)); + EXPECT_EQ(2, DecisionTree::Traverse(tree_config, 0, *++example_it)); + } } TEST_F(DecisionTreeTest, TraverseCategoricalIdBinarySplit) { diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc index 12b377dda7852bb5a580c4ccc1d239709ef9bfc0..cf4f9a097a3368465fd4d9afb981bbaa68b4df49 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc @@ -94,10 +94,6 @@ Status BatchFeatures::Initialize( shape_flat(0) == batch_size_, errors::InvalidArgument( "Sparse float feature shape incompatible with batch size.")); - TF_CHECK_AND_RETURN_IF_ERROR( - shape_flat(1) <= 1, - errors::InvalidArgument( - "Sparse float features may not be multi-valent.")); auto tensor_shape = TensorShape({shape_flat(0), shape_flat(1)}); auto order_dims = sparse::SparseTensor::VarDimArray({0, 1}); sparse_float_feature_columns_.emplace_back(sparse_float_feature_indices, diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h index bb11dc9a0778c062c68433c001e7935388e0f45c..7a550d6f7328765d8815a947885e47fa0b0a8f8b 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h @@ -45,6 +45,22 @@ class BatchFeatures { std::vector sparse_int_feature_values_list, std::vector sparse_int_feature_shapes_list); + Status GetFeatureColumnSizes(int64* const num_dense_float_features, + int64* const num_sparse_float_features, + int64* const num_sparse_int_features) const { + QCHECK_NE(num_dense_float_features, nullptr); + QCHECK_NE(num_sparse_float_features, nullptr); + QCHECK_NE(num_sparse_int_features, nullptr); + *num_dense_float_features = dense_float_feature_columns_.size(); + *num_sparse_float_features = sparse_float_feature_columns_.size(); + *num_sparse_int_features = sparse_int_feature_columns_.size(); + if (*num_dense_float_features == 0 && *num_sparse_float_features == 0 && + *num_sparse_int_features == 0) { + return errors::FailedPrecondition("Not intialized yet."); + } + return Status::OK(); + } + // Creates an example iterable for the requested slice. ExamplesIterable examples_iterable(int64 example_start, int64 example_end) const { diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc index 7f523d527adeb60d179bfce4bc5ef32e75e34ca2..9de3e32b097a151b3bd6f5c30df2db0938b65e9c 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc @@ -129,19 +129,6 @@ TEST_F(BatchFeaturesTest, SparseFloatFeatures_IncompatibleShape) { {sparse_float_feature_shape}, {}, {}, {})); } -TEST_F(BatchFeaturesTest, SparseFloatFeatures_Multivalent) { - BatchFeatures batch_features(2); - auto sparse_float_feature_indices = AsTensor({0, 0, 1, 0}, {2, 2}); - auto sparse_float_feature_values = AsTensor({3.0f, 7.0f}); - auto sparse_float_feature_shape = AsTensor({2, 2}); - auto expected_error = - InvalidArgument("Sparse float features may not be multi-valent."); - EXPECT_EQ(expected_error, batch_features.Initialize( - {}, {sparse_float_feature_indices}, - {sparse_float_feature_values}, - {sparse_float_feature_shape}, {}, {}, {})); -} - TEST_F(BatchFeaturesTest, SparseIntFeatures_WrongShapeIndices) { BatchFeatures batch_features(2); auto sparse_int_feature_indices = AsTensor({0, 0, 1, 0}); diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example.h b/tensorflow/contrib/boosted_trees/lib/utils/example.h index 4681eb06aa2c11a33db4d6e8ff3f0148ffd82917..e388cf332c3ff327f79ea57e3a0bccbbaa1b5e45 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/example.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/example.h @@ -16,6 +16,7 @@ #ifndef THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ #define THIRD_PARTY_TENSORFLOW_CONTRIB_BOOSTED_TREES_LIB_UTILS_EXAMPLE_H_ +#include #include #include #include "tensorflow/contrib/boosted_trees/lib/utils/optional_value.h" @@ -23,6 +24,86 @@ namespace tensorflow { namespace boosted_trees { namespace utils { +// Represents sparse vector that have a value for some feature indices within +// the feature column. +// Allows subscript access []. +template +class SparseMultidimensionalValues { + public: + void Add(const int32 feature_idx, const T value) { + values_.emplace_back(feature_idx, value); + } + + void Clear() { values_.clear(); } + + void Reserve(const int32 size) { values_.reserve(size); } + + OptionalValue operator[](int feature_idx) const { + auto value_iter = + std::find_if(values_.begin(), values_.end(), + [&feature_idx](const std::pair& element) { + return element.first == feature_idx; + }); + + if (value_iter == values_.end()) { + return OptionalValue(); + } + return OptionalValue(value_iter->second); + } + + private: + std::vector> values_; +}; + +// Represents storage for a sparse float feature column. Can store values either +// for one dimensional or a multivalent (multidimensional) sparse column. +// Allows subscript operator access [feature_id]. +template +class SparseFloatFeatureColumn { + public: + void Reserve(const int32 size) { + if (!single_dimensional_) { + mutlidimensional_values.Reserve(size); + } + } + + void SetDimension(const int32 dimension) { + single_dimensional_ = dimension <= 1; + } + + void Add(const int32 feature_idx, const float value) { + if (single_dimensional_) { + DCHECK_EQ(0, feature_idx); + single_value_ = value; + } else { + mutlidimensional_values.Add(feature_idx, value); + } + initialized_ = true; + } + + void Clear() { + single_dimensional_ = false; + initialized_ = false; + mutlidimensional_values.Clear(); + } + + OptionalValue operator[](int feature_idx) const { + if (!initialized_) { + return OptionalValue(); + } + if (single_dimensional_) { + return OptionalValue(single_value_); + } else { + return mutlidimensional_values[feature_idx]; + } + } + + private: + bool single_dimensional_; + bool initialized_; + T single_value_; + SparseMultidimensionalValues mutlidimensional_values; +}; // Holds data for one example and enables lookup by feature column. struct Example { @@ -35,7 +116,10 @@ struct Example { // Dense and sparse float features indexed by feature column. // TODO(salehay): figure out a design to support multivalent float features. std::vector dense_float_features; - std::vector> sparse_float_features; + + // Sparse float features columns (can be either single or multivalent + // (multidimensional). + std::vector> sparse_float_features; // Sparse integer features indexed by feature column. // Note that all integer features are assumed to be categorical, i.e. will diff --git a/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..be9d63ee8ae426d2d2573e7c156c62e2a3b094e1 --- /dev/null +++ b/tensorflow/contrib/boosted_trees/lib/utils/example_test.cc @@ -0,0 +1,94 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= +#include "tensorflow/contrib/boosted_trees/lib/utils/example.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace boosted_trees { +namespace utils { +namespace { + +class ExampleTest : public ::testing::Test {}; + +TEST_F(ExampleTest, TestSparseMatrix) { + // Create the following matrix (FC is feature column): + // FC | f0 | f1 | f2 + // multidimensional + // 0 | | 0.4 | 0.3 + // 1 | 1 | | 2 + // 2 | 3 | 1 | 5 + // 3 | | | + // one dimensional columns + // 4 | -4 + // 5 | + std::vector> matrix; + matrix.resize(6); + matrix[0].SetDimension(3); + matrix[1].SetDimension(3); + matrix[2].SetDimension(3); + matrix[3].SetDimension(3); + matrix[4].SetDimension(1); + matrix[5].SetDimension(1); + + matrix[0].Add(1, 0.4f); + matrix[0].Add(2, 0.3f); + matrix[1].Add(0, 1.f); + matrix[1].Add(2, 2.f); + matrix[2].Add(0, 3.f); + matrix[2].Add(1, 1.f); + matrix[2].Add(2, 5.f); + matrix[4].Add(0, -4.f); + + // Row 0. + EXPECT_FALSE(matrix[0][0].has_value()); + EXPECT_TRUE(matrix[0][1].has_value()); + EXPECT_EQ(0.4f, matrix[0][1].get_value()); + EXPECT_TRUE(matrix[0][2].has_value()); + EXPECT_EQ(0.3f, matrix[0][2].get_value()); + + // Row 1. + EXPECT_TRUE(matrix[1][0].has_value()); + EXPECT_EQ(1.f, matrix[1][0].get_value()); + EXPECT_FALSE(matrix[1][1].has_value()); + EXPECT_TRUE(matrix[1][2].has_value()); + EXPECT_EQ(2.f, matrix[1][2].get_value()); + + // Row 2. + EXPECT_TRUE(matrix[2][0].has_value()); + EXPECT_EQ(3.f, matrix[2][0].get_value()); + EXPECT_TRUE(matrix[2][1].has_value()); + EXPECT_EQ(1.f, matrix[2][1].get_value()); + EXPECT_TRUE(matrix[2][2].has_value()); + EXPECT_EQ(5.f, matrix[2][2].get_value()); + + // Row 3. + EXPECT_FALSE(matrix[3][0].has_value()); + EXPECT_FALSE(matrix[3][1].has_value()); + EXPECT_FALSE(matrix[3][2].has_value()); + + // Row 4. + EXPECT_TRUE(matrix[4][0].has_value()); + EXPECT_EQ(-4.f, matrix[4][0].get_value()); + + // Row 5. + EXPECT_FALSE(matrix[5][0].has_value()); +} + +} // namespace +} // namespace utils +} // namespace boosted_trees +} // namespace tensorflow diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc index c73dc8e15d42f2c80078cf628b5cd5773f5860ff..e7e0b568c6f3b100969c5a6263fd0c36c7803f9f 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.cc @@ -36,12 +36,14 @@ ExamplesIterable::ExamplesIterable( // Create sparse float column iterables and values. sparse_float_column_iterables_.reserve(sparse_float_feature_columns.size()); sparse_float_column_values_.reserve(sparse_float_feature_columns.size()); + sparse_float_dimensions_.reserve(sparse_float_feature_columns.size()); for (auto& sparse_float_column : sparse_float_feature_columns) { sparse_float_column_iterables_.emplace_back( sparse_float_column.indices().template matrix(), example_start, example_end); sparse_float_column_values_.emplace_back( sparse_float_column.values().template vec()); + sparse_float_dimensions_.push_back(sparse_float_column.shape()[1]); } // Create sparse int column iterables and values. @@ -73,9 +75,9 @@ Iterator::Iterator(ExamplesIterable* iter, int64 example_idx) // Pre-size example features. example_.dense_float_features.resize( iter_->dense_float_column_values_.size()); + example_.sparse_int_features.resize(iter_->sparse_int_column_values_.size()); example_.sparse_float_features.resize( iter_->sparse_float_column_values_.size()); - example_.sparse_int_features.resize(iter_->sparse_int_column_values_.size()); } } // namespace utils diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h index 67efb82a227a3d7e92cdf5c8307a6f04c45fb617..5b33c8158879ec65425ac77b5338ee98fbdf07db 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable.h @@ -87,19 +87,52 @@ class ExamplesIterable { // Get sparse float values per column. auto& sparse_float_features = example_.sparse_float_features; + // Iterate through each sparse float feature column. for (size_t sparse_float_idx = 0; - sparse_float_idx < sparse_float_features.size(); + sparse_float_idx < iter_->sparse_float_column_iterables_.size(); ++sparse_float_idx) { + // Clear info from a previous instance. + sparse_float_features[sparse_float_idx].Clear(); + + // Get range for values tensor. const auto& row_range = (*sparse_float_column_iterators_[sparse_float_idx]); DCHECK_EQ(example_idx_, row_range.example_idx); + + // If the example has this feature column. if (row_range.start < row_range.end) { - DCHECK_EQ(1, row_range.end - row_range.start); - sparse_float_features[sparse_float_idx] = OptionalValue( - iter_->sparse_float_column_values_[sparse_float_idx]( - row_range.start)); - } else { - sparse_float_features[sparse_float_idx] = OptionalValue(); + const int32 dimension = + iter_->sparse_float_dimensions_[sparse_float_idx]; + sparse_float_features[sparse_float_idx].SetDimension(dimension); + if (dimension <= 1) { + // single dimensional sparse feature column. + DCHECK_EQ(1, row_range.end - row_range.start); + sparse_float_features[sparse_float_idx].Add( + 0, iter_->sparse_float_column_values_[sparse_float_idx]( + row_range.start)); + } else { + // Retrieve original indices tensor. + const TTypes::ConstMatrix& indices = + iter_->sparse_float_column_iterables_[sparse_float_idx] + .sparse_indices(); + + sparse_float_features[sparse_float_idx].Reserve(row_range.end - + row_range.start); + + // For each value. + for (int64 row_idx = row_range.start; row_idx < row_range.end; + ++row_idx) { + // Get the feature id for the feature column and the value. + const int32 feature_id = indices(row_idx, 1); + DCHECK_EQ(example_idx_, indices(row_idx, 0)); + + // Save the value to our sparse matrix. + sparse_float_features[sparse_float_idx].Add( + feature_id, + iter_->sparse_float_column_values_[sparse_float_idx]( + row_idx)); + } + } } } @@ -158,6 +191,9 @@ class ExamplesIterable { // Sparse float column values. std::vector::ConstVec> sparse_float_column_values_; + // Dimensions for sparse float feature columns. + std::vector sparse_float_dimensions_; + // Sparse int column iterables. std::vector sparse_int_column_iterables_; diff --git a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc index d93bcc8aa67102fcdacf130d90769514ce6c8170..d8a608864834b17886313a368221fbf94e31c98e 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/examples_iterable_test.cc @@ -26,17 +26,17 @@ class ExamplesIterableTest : public ::testing::Test {}; TEST_F(ExamplesIterableTest, Iterate) { // Create a batch of 8 examples having one dense float, two sparse float and - // two sparse int features. + // two sparse int features. Second sparse float feature is multivalent. // The data looks like the following: // Instance | DenseF1 | SparseF1 | SparseF2 | SparseI1 | SparseI2 | - // 0 | 7 | -3 | | 1, 8 | | - // 1 | -2 | | 4 | 0 | 7 | - // 2 | 8 | 0 | | | 13 | - // 3 | 1 | 5 | 7 | 2, 0 | 4 | - // 4 | 0 | 0 | | | 0 | - // 5 | -4 | | 9 | | | - // 6 | 7 | | | | | - // 7 | -2 | | -4 | 5 | | + // 0 | 7 | -3 | | 1 | 1, 8 | | + // 1 | -2 | | 4 | | 0 | 7 | + // 2 | 8 | 0 | | 3 | | 13 | + // 3 | 1 | 5 | 7 | | 2, 0 | 4 | + // 4 | 0 | 0 | | 4.3 | | 0 | + // 5 | -4 | | 9 | 0.8 | | | + // 6 | 7 | | | | | | + // 7 | -2 | | -4 | | 5 | | auto dense_float_tensor = test::AsTensor( {7.0f, -2.0f, 8.0f, 1.0f, 0.0f, -4.0f, 7.0f, -2.0f}, {8, 1}); auto sparse_float_indices1 = @@ -45,10 +45,11 @@ TEST_F(ExamplesIterableTest, Iterate) { auto sparse_float_shape1 = TensorShape({8, 1}); sparse::SparseTensor sparse_float_tensor1( sparse_float_indices1, sparse_float_values1, sparse_float_shape1); - auto sparse_float_indices2 = - test::AsTensor({1, 0, 3, 0, 5, 0, 7, 0}, {4, 2}); - auto sparse_float_values2 = test::AsTensor({4.0f, 7.0f, 9.0f, -4.0f}); - auto sparse_float_shape2 = TensorShape({8, 1}); + auto sparse_float_indices2 = test::AsTensor( + {0, 1, 1, 0, 2, 1, 3, 0, 4, 1, 5, 0, 5, 1, 7, 0}, {8, 2}); + auto sparse_float_values2 = + test::AsTensor({1.f, 4.0f, 3.f, 7.0f, 4.3f, 9.0f, 0.8f, -4.0f}); + auto sparse_float_shape2 = TensorShape({8, 2}); sparse::SparseTensor sparse_float_tensor2( sparse_float_indices2, sparse_float_values2, sparse_float_shape2); auto sparse_int_indices1 = @@ -67,15 +68,19 @@ TEST_F(ExamplesIterableTest, Iterate) { auto validate_example_features = [](int64 example_idx, const Example& example) { EXPECT_EQ(1, example.dense_float_features.size()); - EXPECT_EQ(2, example.sparse_float_features.size()); switch (example_idx) { case 0: { EXPECT_EQ(0, example.example_idx); EXPECT_EQ(7.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(-3.0f, example.sparse_float_features[0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(-3.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2 - multivalent. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(1.0f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(2, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(1)); EXPECT_EQ(1, example.sparse_int_features[0].count(8)); @@ -84,9 +89,13 @@ TEST_F(ExamplesIterableTest, Iterate) { case 1: { EXPECT_EQ(1, example.example_idx); EXPECT_EQ(-2.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(4.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(4.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(1, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(0)); EXPECT_EQ(1, example.sparse_int_features[1].size()); @@ -95,9 +104,14 @@ TEST_F(ExamplesIterableTest, Iterate) { case 2: { EXPECT_EQ(2, example.example_idx); EXPECT_EQ(8.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(0.0f, example.sparse_float_features[0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(3.f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[1].size()); EXPECT_EQ(1, example.sparse_int_features[1].count(13)); @@ -105,10 +119,14 @@ TEST_F(ExamplesIterableTest, Iterate) { case 3: { EXPECT_EQ(3, example.example_idx); EXPECT_EQ(1.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(5.0f, example.sparse_float_features[0].get_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(7.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(5.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(7.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(2, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(2)); EXPECT_EQ(1, example.sparse_int_features[0].count(0)); @@ -118,9 +136,14 @@ TEST_F(ExamplesIterableTest, Iterate) { case 4: { EXPECT_EQ(4, example.example_idx); EXPECT_EQ(0.0f, example.dense_float_features[0]); - EXPECT_TRUE(example.sparse_float_features[0].has_value()); - EXPECT_EQ(0.0f, example.sparse_float_features[0].get_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_TRUE(example.sparse_float_features[0][0].has_value()); + EXPECT_EQ(0.0f, example.sparse_float_features[0][0].get_value()); + // SparseF2. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(4.3f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[1].size()); EXPECT_EQ(1, example.sparse_int_features[1].count(0)); @@ -128,24 +151,37 @@ TEST_F(ExamplesIterableTest, Iterate) { case 5: { EXPECT_EQ(5, example.example_idx); EXPECT_EQ(-4.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(9.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(9.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_TRUE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(0.8f, example.sparse_float_features[1][1].get_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); } break; case 6: { EXPECT_EQ(6, example.example_idx); EXPECT_EQ(7.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_FALSE(example.sparse_float_features[1].has_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_FALSE(example.sparse_float_features[1][0].has_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(0, example.sparse_int_features[0].size()); } break; case 7: { EXPECT_EQ(7, example.example_idx); EXPECT_EQ(-2.0f, example.dense_float_features[0]); - EXPECT_FALSE(example.sparse_float_features[0].has_value()); - EXPECT_TRUE(example.sparse_float_features[1].has_value()); - EXPECT_EQ(-4.0f, example.sparse_float_features[1].get_value()); + // SparseF1. + EXPECT_FALSE(example.sparse_float_features[0][0].has_value()); + // SparseF2. + EXPECT_TRUE(example.sparse_float_features[1][0].has_value()); + EXPECT_EQ(-4.0f, example.sparse_float_features[1][0].get_value()); + EXPECT_FALSE(example.sparse_float_features[1][1].has_value()); + EXPECT_EQ(1, example.sparse_int_features[0].size()); EXPECT_EQ(1, example.sparse_int_features[0].count(5)); } break; @@ -158,6 +194,7 @@ TEST_F(ExamplesIterableTest, Iterate) { {dense_float_tensor}, {sparse_float_tensor1, sparse_float_tensor2}, {sparse_int_tensor1, sparse_int_tensor2}, 0, 8); int64 example_idx = 0; + for (const auto& example : full_iterable) { validate_example_features(example_idx, example); ++example_idx; diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h index 78a5752730cb793394c41c56ab83b084a6f76088..9664c9d1c6a0c0c8b1bbd1506944c54d2310c611 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h +++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable.h @@ -112,6 +112,8 @@ class SparseColumnIterable { int64 example_start() const { return example_start_; } int64 example_end() const { return example_end_; } + const TTypes::ConstMatrix& sparse_indices() const { return ix_; } + private: // Sparse indices matrix. TTypes::ConstMatrix ix_; diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc index 7792bd8c66c53c0f11cff113c3e5526c6d50dbb8..0138aae3dbd3773241cb6644db625b99f9bf1372 100644 --- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc +++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc @@ -34,19 +34,19 @@ TEST_F(SparseColumnIterableTest, Empty) { } TEST_F(SparseColumnIterableTest, Iterate) { - // 8 examples having 7 sparse features with the third multi-valent. + // 8 examples having 7 sparse features with the 3rd and 7th multi-valent. // This can be visualized like the following: // Instance | Sparse | - // 0 | x | + // 0 | x | // 1 | | // 2 | | // 3 | xxx | - // 4 | x | + // 4 | x | // 5 | | // 6 | | - // 7 | xx | + // 7 | x x | const auto indices = - AsTensor({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 1}, {7, 2}); + AsTensor({0, 0, 3, 0, 3, 1, 3, 2, 4, 0, 7, 0, 7, 2}, {7, 2}); auto validate_example_range = [](const ExampleRowRange& range) { switch (range.example_idx) { diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto index 2e9d45efd71adef828a55e54f48d2740b8c1a12e..f14abf45a517ad7c4c6d7bb1ab88b7a1d47d6fb6 100644 --- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto +++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto @@ -53,6 +53,9 @@ message DenseFloatBinarySplit { // Float feature column and split threshold describing // the rule feature <= threshold. int32 feature_column = 1; + // If feature column is multivalent, this holds the index of the feature for + // the split. Defaults to 0. + int32 feature_id = 5; float threshold = 2; // Node children indexing into a contiguous diff --git a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py index f8f4b43a072a91f1563b20d6ba3aef82fd4b9896..5a917ca42897a263bf9f868393453ba232745e65 100644 --- a/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py +++ b/tensorflow/contrib/boosted_trees/python/training/functions/gbdt_batch.py @@ -344,7 +344,7 @@ class GradientBoostedDecisionTreeModel(object): learner_config.num_classes == 2) def _predict_and_return_dict(self, ensemble_handle, ensemble_stamp, mode): - """Runs prediciton and returns a dictionary of the prediction results. + """Runs prediction and returns a dictionary of the prediction results. Args: ensemble_handle: ensemble resource handle. diff --git a/tensorflow/contrib/boosted_trees/python/utils/losses.py b/tensorflow/contrib/boosted_trees/python/utils/losses.py index 4f128b230180d8e8070f63c369bc7fc2f3d24376..1e8b3ac08a74a94a0e5729e42ace91398a7b5c94 100644 --- a/tensorflow/contrib/boosted_trees/python/utils/losses.py +++ b/tensorflow/contrib/boosted_trees/python/utils/losses.py @@ -101,7 +101,10 @@ def per_example_maxent_loss(labels, weights, logits, num_classes, eps=1e-15): unweighted_loss = array_ops.expand_dims(-math_ops.log(probs_for_real_class), 1) - return unweighted_loss * weights, control_flow_ops.no_op() + if weights is None: + return unweighted_loss, control_flow_ops.no_op() + else: + return unweighted_loss * weights, control_flow_ops.no_op() def per_example_squared_loss(labels, weights, predictions): diff --git a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h index 77e6ecb443dd3f0f7a96b7453f558d58f01c7a21..284ad5cdb9abf374650940ade7bb36663d72c0dd 100644 --- a/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h +++ b/tensorflow/contrib/boosted_trees/resources/decision_tree_ensemble_resource.h @@ -47,6 +47,7 @@ class DecisionTreeEnsembleResource : public StampedResource { int32 num_trees() const { return decision_tree_ensemble_->trees_size(); } bool InitFromSerialized(const string& serialized, const int64 stamp_token) { + CHECK_EQ(stamp(), -1) << "Must Reset before Init."; if (ParseProtoUnlimited(decision_tree_ensemble_, serialized)) { set_stamp(stamp_token); return true; @@ -126,7 +127,7 @@ class DecisionTreeEnsembleResource : public StampedResource { // Resets the resource and frees the protos in arena. // Caller needs to hold the mutex lock while calling this. - void Reset() { + virtual void Reset() { // Reset stamp. set_stamp(-1); diff --git a/tensorflow/contrib/cloud/BUILD b/tensorflow/contrib/cloud/BUILD index eec2beddc487d67171ea43b0e46e7c8f7c11a4f3..aa8f5ed12bc6f779e3c1a923b9225ec283189747 100644 --- a/tensorflow/contrib/cloud/BUILD +++ b/tensorflow/contrib/cloud/BUILD @@ -63,11 +63,15 @@ tf_py_test( ":bigquery_reader_ops_op_lib", ":cloud_py", "//tensorflow/contrib/cloud/kernels:bigquery_reader_ops", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:data_flow_ops", + "//tensorflow/python:dtypes", "//tensorflow/python:io_ops", "//tensorflow/python:parsing_ops", + "//tensorflow/python:platform", + "//tensorflow/python:util", ], tags = ["manual"], ) diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD index 09ec7e42c7eede97b9c7eeee329fe0649365869e..56f930a9a8d32c5c3a025163ef56c9562f17d864 100644 --- a/tensorflow/contrib/cloud/kernels/BUILD +++ b/tensorflow/contrib/cloud/kernels/BUILD @@ -23,7 +23,9 @@ load( filegroup( name = "all_files", srcs = glob( - ["**/*"], + include = [ + "**/*", + ], exclude = [ "**/METADATA", "**/OWNERS", @@ -34,9 +36,7 @@ filegroup( tf_kernel_library( name = "bigquery_reader_ops", - srcs = [ - "bigquery_reader_ops.cc", - ], + srcs = ["bigquery_reader_ops.cc"], visibility = ["//visibility:public"], deps = [ ":bigquery_table_accessor", @@ -50,12 +50,8 @@ tf_kernel_library( cc_library( name = "bigquery_table_accessor", - srcs = [ - "bigquery_table_accessor.cc", - ], - hdrs = [ - "bigquery_table_accessor.h", - ], + srcs = ["bigquery_table_accessor.cc"], + hdrs = ["bigquery_table_accessor.h"], copts = tf_copts(), linkstatic = 1, deps = [ @@ -64,7 +60,6 @@ cc_library( "//tensorflow/core:protos_all_cc", "//tensorflow/core/platform/cloud:curl_http_request", "//tensorflow/core/platform/cloud:google_auth_provider", - "//tensorflow/core/platform/cloud:http_request", ], alwayslink = 1, ) @@ -88,8 +83,6 @@ tf_cc_test( tf_proto_library( name = "bigquery_table_partition_proto", - srcs = [ - "bigquery_table_partition.proto", - ], + srcs = ["bigquery_table_partition.proto"], cc_api_version = 2, ) diff --git a/tensorflow/contrib/cluster_resolver/BUILD b/tensorflow/contrib/cluster_resolver/BUILD index 9501c332454238e0c4eb36d25e97f06dde9abed5..15abd2be0385eb776ff4f76484133efb6e34f076 100644 --- a/tensorflow/contrib/cluster_resolver/BUILD +++ b/tensorflow/contrib/cluster_resolver/BUILD @@ -13,7 +13,9 @@ licenses(["notice"]) # Apache 2.0 filegroup( name = "all_files", srcs = glob( - ["**/*"], + include = [ + "**/*", + ], exclude = [ "**/METADATA", "**/OWNERS", @@ -37,9 +39,7 @@ py_library( py_library( name = "cluster_resolver_py", - srcs = [ - "python/training/cluster_resolver.py", - ], + srcs = ["python/training/cluster_resolver.py"], srcs_version = "PY2AND3", deps = [ "//tensorflow/python:training", @@ -48,9 +48,7 @@ py_library( py_library( name = "gce_cluster_resolver_py", - srcs = [ - "python/training/gce_cluster_resolver.py", - ], + srcs = ["python/training/gce_cluster_resolver.py"], srcs_version = "PY2AND3", deps = [ ":cluster_resolver_py", @@ -60,9 +58,7 @@ py_library( py_library( name = "tpu_cluster_resolver_py", - srcs = [ - "python/training/tpu_cluster_resolver.py", - ], + srcs = ["python/training/tpu_cluster_resolver.py"], srcs_version = "PY2AND3", deps = [ ":cluster_resolver_py", @@ -79,6 +75,7 @@ tf_py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:training", ], main = "python/training/cluster_resolver_test.py", ) @@ -88,11 +85,13 @@ tf_py_test( size = "small", srcs = ["python/training/gce_cluster_resolver_test.py"], additional_deps = [ + ":cluster_resolver_py", ":gce_cluster_resolver_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:training", ], main = "python/training/gce_cluster_resolver_test.py", ) @@ -107,6 +106,7 @@ tf_py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:training", ], main = "python/training/tpu_cluster_resolver_test.py", ) diff --git a/tensorflow/contrib/cmake/external/cub.cmake b/tensorflow/contrib/cmake/external/cub.cmake index 7b263806d733f0e1deafe3e8fdd9baf2bb6fd81f..836889895567f679d9960e29ece1600d1a7a58eb 100644 --- a/tensorflow/contrib/cmake/external/cub.cmake +++ b/tensorflow/contrib/cmake/external/cub.cmake @@ -14,7 +14,7 @@ # ============================================================================== include (ExternalProject) -set(cub_URL https://github.com/NVlabs/cub/archive/1.7.4.zip) +set(cub_URL https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip) set(cub_HASH SHA256=20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31) set(cub_BUILD ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) set(cub_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cub/src/cub) diff --git a/tensorflow/contrib/cmake/external/gif.cmake b/tensorflow/contrib/cmake/external/gif.cmake index 5cb719b8787781084335779960887613df90217d..3d53c51fffcec1602a3b5553cdf3b225e3b0ae46 100644 --- a/tensorflow/contrib/cmake/external/gif.cmake +++ b/tensorflow/contrib/cmake/external/gif.cmake @@ -15,7 +15,7 @@ include (ExternalProject) set(gif_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/gif_archive/giflib-5.1.4/) -set(gif_URL http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz) +set(gif_URL https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz) set(gif_HASH SHA256=34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1) set(gif_INSTALL ${CMAKE_BINARY_DIR}/gif/install) set(gif_BUILD ${CMAKE_BINARY_DIR}/gif/src/gif) diff --git a/tensorflow/contrib/cmake/external/jpeg.cmake b/tensorflow/contrib/cmake/external/jpeg.cmake index 058f554b8f2ffc4f925012e8772c684965304833..d9a165e856c588880ebdf996666d70c9e7f53da8 100644 --- a/tensorflow/contrib/cmake/external/jpeg.cmake +++ b/tensorflow/contrib/cmake/external/jpeg.cmake @@ -15,7 +15,7 @@ include (ExternalProject) set(jpeg_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/jpeg_archive) -set(jpeg_URL http://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz) +set(jpeg_URL https://mirror.bazel.build/www.ijg.org/files/jpegsrc.v9a.tar.gz) set(jpeg_HASH SHA256=3a753ea48d917945dd54a2d97de388aa06ca2eb1066cbfdc6652036349fe05a7) set(jpeg_BUILD ${CMAKE_CURRENT_BINARY_DIR}/jpeg/src/jpeg) set(jpeg_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/jpeg/install) diff --git a/tensorflow/contrib/cmake/external/lmdb.cmake b/tensorflow/contrib/cmake/external/lmdb.cmake index 28ec833babe8f8e600c7c0179dff511ce4d26105..79971b7cfc3c72e4b6290ccb71d40a20d1180c01 100644 --- a/tensorflow/contrib/cmake/external/lmdb.cmake +++ b/tensorflow/contrib/cmake/external/lmdb.cmake @@ -15,7 +15,7 @@ include (ExternalProject) set(lmdb_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/lmdb) -set(lmdb_URL http://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz) +set(lmdb_URL https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz) set(lmdb_HASH SHA256=108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326) set(lmdb_BUILD ${CMAKE_BINARY_DIR}/lmdb/src/lmdb) set(lmdb_INSTALL ${CMAKE_BINARY_DIR}/lmdb/install) diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake index d600d8c3c0d30ec517d0abc4bac94c588b5268d4..1e300e21df17eeee0abfc2becdab746fbfc62ff6 100644 --- a/tensorflow/contrib/cmake/external/protobuf.cmake +++ b/tensorflow/contrib/cmake/external/protobuf.cmake @@ -15,8 +15,8 @@ include (ExternalProject) set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src) -set(PROTOBUF_URL https://github.com/mrry/protobuf.git) # Includes MSVC fix. -set(PROTOBUF_TAG 1d2c7b6c7376f396c8c7dd9b6afd2d4f83f3cb05) +set(PROTOBUF_URL https://github.com/google/protobuf.git) +set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9) if(WIN32) set(protobuf_STATIC_LIBRARIES diff --git a/tensorflow/contrib/cmake/external/snappy.cmake b/tensorflow/contrib/cmake/external/snappy.cmake index a35d8654fb6fa5f5b5d230ffbc061d050e5aeb5e..2d2451521c0f9127e2c76e6270694ac21fe8db93 100644 --- a/tensorflow/contrib/cmake/external/snappy.cmake +++ b/tensorflow/contrib/cmake/external/snappy.cmake @@ -47,4 +47,4 @@ ExternalProject_Add(snappy ) # actually enables snappy in the source code -add_definitions(-DSNAPPY) \ No newline at end of file +add_definitions(-DTF_USE_SNAPPY) diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake index 1b64a52ecef062f9b7ef28c2b427e95b98279d08..c3dc8531bb9f0164f06841d9715f227202fdb7c9 100644 --- a/tensorflow/contrib/cmake/tf_core_framework.cmake +++ b/tensorflow/contrib/cmake/tf_core_framework.cmake @@ -191,6 +191,10 @@ file(GLOB_RECURSE tf_core_lib_srcs "${tensorflow_source_dir}/tensorflow/core/lib/*.h" "${tensorflow_source_dir}/tensorflow/core/lib/*.cc" "${tensorflow_source_dir}/tensorflow/core/public/*.h" + # TODO(@jart): Move StatusOr into core. + "${tensorflow_source_dir}/tensorflow/compiler/xla/statusor.cc" + "${tensorflow_source_dir}/tensorflow/compiler/xla/statusor.h" + "${tensorflow_source_dir}/tensorflow/compiler/xla/statusor_internals.h" ) file(GLOB tf_core_platform_srcs diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake index 65565aad7ea5469926f320839455cd884a343713..f978c8ccd5a454ca4a89de0ab5d757b566295c60 100644 --- a/tensorflow/contrib/cmake/tf_core_kernels.cmake +++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake @@ -69,6 +69,8 @@ if(tensorflow_BUILD_CONTRIB_KERNELS) "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/training_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc" + "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/prefetching_kernels.cc" + "${tensorflow_source_dir}/tensorflow/contrib/data/ops/prefetching_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/clustering_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/masked_matmul_ops.cc" "${tensorflow_source_dir}/tensorflow/contrib/factorization/kernels/wals_solver_ops.cc" diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake index dc9973917e48e77a0ffe04a687cb205e6342f46a..4a61ed7a3548b1992ddc71acb8a7761e252296ea 100644 --- a/tensorflow/contrib/cmake/tf_core_ops.cmake +++ b/tensorflow/contrib/cmake/tf_core_ops.cmake @@ -13,6 +13,7 @@ # limitations under the License. # ============================================================================== set(tf_op_lib_names + "audio_ops" "array_ops" "bitwise_ops" "candidate_sampling_ops" @@ -43,6 +44,7 @@ set(tf_op_lib_names "state_ops" "stateless_random_ops" "string_ops" + "summary_ops" "training_ops" ) @@ -79,6 +81,7 @@ GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_prediction "${tensorflow_source_dir}/t GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_quantiles "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_stats_accumulator "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(cudnn_rnn "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc") +GENERATE_CONTRIB_OP_LIBRARY(data_prefetching "${tensorflow_source_dir}/tensorflow/contrib/data/ops/prefetching_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(factorization_clustering "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/clustering_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(factorization_factorization "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/factorization_ops.cc") GENERATE_CONTRIB_OP_LIBRARY(framework_variable "${tensorflow_source_dir}/tensorflow/contrib/framework/ops/variable_ops.cc") diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake index e83618a94ecea28a46bab0ab7b3d8e2517102823..277818b159062da4ba6efaacbe006da623c8619c 100755 --- a/tensorflow/contrib/cmake/tf_python.cmake +++ b/tensorflow/contrib/cmake/tf_python.cmake @@ -347,6 +347,8 @@ add_python_module("tensorflow/contrib/distributions/python") add_python_module("tensorflow/contrib/distributions/python/kernel_tests") add_python_module("tensorflow/contrib/distributions/python/ops") add_python_module("tensorflow/contrib/distributions/python/ops/bijectors") +add_python_module("tensorflow/contrib/eager") +add_python_module("tensorflow/contrib/eager/python") add_python_module("tensorflow/contrib/estimator") add_python_module("tensorflow/contrib/estimator/python") add_python_module("tensorflow/contrib/estimator/python/estimator") @@ -640,6 +642,7 @@ add_python_module("tensorflow/contrib/reduce_slice_ops/ops") add_python_module("tensorflow/contrib/reduce_slice_ops/python") add_python_module("tensorflow/contrib/reduce_slice_ops/python/kernel_tests") add_python_module("tensorflow/contrib/reduce_slice_ops/python/ops") +add_python_module("tensorflow/contrib/summary") # Generate the tensorflow.python.platform.build_info module. set(BUILD_INFO_PY "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/platform/build_info.py") @@ -766,6 +769,8 @@ GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_stats_accumulator_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_stats_accumulator_ops.py) GENERATE_PYTHON_OP_LIB("contrib_cudnn_rnn_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/cudnn_rnn/ops/gen_cudnn_rnn_ops.py) +GENERATE_PYTHON_OP_LIB("contrib_data_prefetching_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/data/python/ops/gen_prefetching_ops.py) GENERATE_PYTHON_OP_LIB("contrib_factorization_clustering_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/factorization/python/ops/gen_clustering_ops.py) GENERATE_PYTHON_OP_LIB("contrib_factorization_factorization_ops" @@ -812,6 +817,8 @@ GENERATE_PYTHON_OP_LIB("stateless_random_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/stateless/gen_stateless_random_ops.py) GENERATE_PYTHON_OP_LIB("debug_ops" DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/debug/ops/gen_debug_ops.py) +GENERATE_PYTHON_OP_LIB("summary_ops" + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/summary/gen_summary_ops.py) add_custom_target(tf_python_ops SOURCES ${tf_python_ops_generated_files} ${PYTHON_PROTO_GENFILES}) add_dependencies(tf_python_ops tf_python_op_gen_main) @@ -874,6 +881,8 @@ set (pywrap_tensorflow_internal_src "${tensorflow_source_dir}/tensorflow/python/lib/io/py_record_writer.cc" "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.h" "${tensorflow_source_dir}/tensorflow/python/util/kernel_registry.cc" + "${tensorflow_source_dir}/tensorflow/python/util/util.h" + "${tensorflow_source_dir}/tensorflow/python/util/util.cc" "${tensorflow_source_dir}/tensorflow/cc/framework/ops.cc" "${tensorflow_source_dir}/tensorflow/cc/framework/scope.cc" "${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.cc" diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake index 85487605332e4940cbd51a6082da72a60b1a9faa..77d21249148cc900a1bb4fc2742956aee47734de 100644 --- a/tensorflow/contrib/cmake/tf_tests.cmake +++ b/tensorflow/contrib/cmake/tf_tests.cmake @@ -194,10 +194,13 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/python/profiler/pprof_profiler_test.py" # flaky test "${tensorflow_source_dir}/tensorflow/python/profiler/internal/run_metadata_test.py" + # Fails because uses data dependencies with bazel "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py" - # flaky tests - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py" # takes very long to run - "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/internal/run_metadata_test.py" + # requires scipy + "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/*_test.py" + "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py" + # Takes very long to run without sharding (defined in bazel build file). + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py" # Loading resources in contrib doesn't seem to work on Windows "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/client/random_forest_test.py" "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/tensor_forest_test.py" @@ -216,14 +219,23 @@ if (tensorflow_BUILD_PYTHON_TESTS) # stl on windows handles overflows different "${tensorflow_source_dir}/tensorflow/python/kernel_tests/as_string_op_test.py" - "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/string_to_number_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/clip_ops_test.py" + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/tensor_array_ops_test.py" # Needs portpicker. # Numerical issues, calculations off. "${tensorflow_source_dir}/tensorflow/python/kernel_tests/concat_op_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py" + "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/wals_test.py" # Float division by zero "${tensorflow_source_dir}/tensorflow/python/kernel_tests/benchmark_test.py" + # Flaky, for unknown reasons. Cannot reproduce in terminal. Revisit once we can get stack traces. + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/batch_matmul_op_test.py" + # Flaky because of local cluster creation. + "${tensorflow_source_dir}/tensorflow/python/training/sync_replicas_optimizer_test.py" + "${tensorflow_source_dir}/tensorflow/python/debug/lib/session_debug_grpc_test.py" + "${tensorflow_source_dir}tensorflow/python/training/localhost_cluster_performance_test.py" + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py" + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py" + "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py" # Type error in testRemoteIteratorUsingRemoteCallOpDirectSessionGPUCPU. "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py" @@ -233,6 +245,7 @@ if (tensorflow_BUILD_PYTHON_TESTS) # Depends on gemmlowp -> pthread "${tensorflow_source_dir}/tensorflow/python/kernel_tests/neon_depthwise_conv_op_test.py" # int32/int64 mixup + "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cast_op_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/variable_scope_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/functional_ops_test.py" "${tensorflow_source_dir}/tensorflow/python/kernel_tests/py_func_test.py" @@ -251,7 +264,6 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/python/kernel_tests/dataset_constructor_op_test.py" # Segfaults on windows "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py" # Segfaults on Windows. "${tensorflow_source_dir}/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py" - "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py" # b/67743142 # Broken tensorboard test due to cmake issues. "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/iterator_ops_cluster_test.py" # Needs portpicker "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py" # b/65430561 @@ -275,6 +287,8 @@ if (tensorflow_BUILD_PYTHON_TESTS) "${tensorflow_source_dir}/tensorflow/python/kernel_tests/spacetodepth_op_test.py" # QuantizeV2 # Windows Path "${tensorflow_source_dir}/tensorflow/contrib/framework/python/ops/checkpoint_ops_test.py" #TODO: Fix path + "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/kmeans_test.py" + "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/estimators/kmeans_test.py" # Numpy upgrade needed? "${tensorflow_source_dir}/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sinh_arcsinh_test.py" # Test should only be run manually diff --git a/tensorflow/contrib/compiler/jit_test.py b/tensorflow/contrib/compiler/jit_test.py index 94aff13a49f5380d5804e190b33613fd42dcaebc..2108e42bce4eba1eed158fe85888f1699a69ba7e 100644 --- a/tensorflow/contrib/compiler/jit_test.py +++ b/tensorflow/contrib/compiler/jit_test.py @@ -173,12 +173,12 @@ class CompilationEnabledInGradientTest(test.TestCase): def testCompilationInGradient(self): with self.test_session(): - x = constant_op.constant(3) - y_nc = math_ops.add(x, x, name="not_compiled") + x = constant_op.constant([[3]]) + y_nc = math_ops.matmul(x, x, name="not_compiled") with jit.experimental_jit_scope(): - y_c = math_ops.add(y_nc, y_nc, name="compiled") + y_c = math_ops.matmul(y_nc, y_nc, name="compiled") x_grads = gradients.gradients([y_c], [x])[0] - operations = x_grads.graph.get_operations() + operations = x.graph.get_operations() c_grad_ops = [ op for op in operations if "gradients/compiled" in op.name] nc_grad_ops = [ @@ -191,19 +191,19 @@ class CompilationEnabledInGradientTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "No attr named"): ncg.get_attr("_XlaCompile") - # d/dx (4 * x) - self.assertAllClose(4, x_grads.eval()) + # d/dx (x ** 4) = 4 * (x ** 3) + self.assertAllClose([[108]], x_grads.eval()) def testCompilationGradientScopeNames(self): with self.test_session(graph=ops.Graph()): with jit.experimental_jit_scope(): # XlaScope 0 - a1 = constant_op.constant(1) - a1t = a1 + a1 + a1 = constant_op.constant([[1]]) + a1t = math_ops.matmul(a1, a1) with jit.experimental_jit_scope(): # XlaScope 1 - a2 = constant_op.constant(1) - a2t = a2 + a2 + a2 = constant_op.constant([[1]]) + a2t = math_ops.matmul(a2, a2) self.assertEqual(b"jit_scope_0", a1.op.get_attr("_XlaScope")) self.assertEqual(b"jit_scope_1", a2.op.get_attr("_XlaScope")) @@ -220,12 +220,12 @@ class CompilationEnabledInGradientTest(test.TestCase): with self.test_session(graph=ops.Graph()): with jit.experimental_jit_scope(True, separate_compiled_gradients=True): # XlaScope 0 - a1 = constant_op.constant(1) - a1t = a1 + a1 + a1 = constant_op.constant([[1]]) + a1t = math_ops.matmul(a1, a1) with jit.experimental_jit_scope(True, separate_compiled_gradients=True): # XlaScope 1 - a2 = constant_op.constant(1) - a2t = a2 + a2 + a2 = constant_op.constant([[1]]) + a2t = math_ops.matmul(a2, a2) self.assertEqual(b"jit_scope_0", a1.op.get_attr("_XlaScope")) self.assertEqual(b"jit_scope_1", a2.op.get_attr("_XlaScope")) diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD index ae9413fdd63cafed306b10a0f68f3fc0315a22c3..f192f78b98174d4e1af2e91f90b6a285fe51b628 100644 --- a/tensorflow/contrib/cudnn_rnn/BUILD +++ b/tensorflow/contrib/cudnn_rnn/BUILD @@ -36,6 +36,7 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "//tensorflow/core:stream_executor", "//tensorflow/core/kernels:bounds_check_lib", "//third_party/eigen3", ], @@ -70,14 +71,23 @@ tf_custom_op_py_library( visibility = ["//visibility:public"], deps = [ ":cudnn_rnn_ops", + "//tensorflow/contrib/rnn:rnn_py", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", + "//tensorflow/python:common_shapes", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers_base", + "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:random_seed", + "//tensorflow/python:rnn_cell", "//tensorflow/python:state_ops", "//tensorflow/python:training", + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", ], ) @@ -104,9 +114,13 @@ tf_custom_op_py_library( "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers_base", "//tensorflow/python:platform", "//tensorflow/python:state_ops", "//tensorflow/python:training", + "//tensorflow/python:util", + "//tensorflow/python:variable_scope", ], ) diff --git a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py index f6c206022c68c4ba78d895f44288f4b180d199c0..3d3f8a3be0554c709ce053106f754f27d8ed630a 100644 --- a/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py +++ b/tensorflow/contrib/cudnn_rnn/python/layers/cudnn_rnn.py @@ -18,7 +18,6 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.cudnn_rnn.python.ops import cudnn_rnn_ops -from tensorflow.contrib.util import loader from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -26,12 +25,8 @@ from tensorflow.python.layers import base as base_layer from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import variable_scope as vs -from tensorflow.python.platform import resource_loader from tensorflow.python.platform import tf_logging as logging -_cudnn_rnn_ops_so = loader.load_op_library( - resource_loader.get_path_to_datafile("_cudnn_rnn_ops.so")) - CUDNN_RNN_UNIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_UNIDIRECTION CUDNN_RNN_BIDIRECTION = cudnn_rnn_ops.CUDNN_RNN_BIDIRECTION CUDNN_LSTM = cudnn_rnn_ops.CUDNN_LSTM diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD index ee96269a739ebb138ea88cf4e192f7925e85447d..eaede0e00ecf1986873d50709d135d3f4b3ac9cd 100644 --- a/tensorflow/contrib/data/BUILD +++ b/tensorflow/contrib/data/BUILD @@ -4,12 +4,20 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load( + "//tensorflow:tensorflow.bzl", + "tf_custom_op_library", + "tf_gen_op_libs", +) + py_library( name = "data", srcs = ["__init__.py"], srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", + "//tensorflow/contrib/data/python/ops:prefetching_py", "//tensorflow/contrib/data/python/ops:readers", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:util", @@ -17,6 +25,20 @@ py_library( ], ) +tf_custom_op_library( + name = "_prefetching_ops.so", + srcs = [ + "ops/prefetching_ops.cc", + ], + deps = [ + "//tensorflow/contrib/data/kernels:prefetching_kernels", + ], +) + +tf_gen_op_libs( + op_lib_names = ["prefetching_ops"], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py index 7ff26e087bbb61963948be1a2edaaa407d0ba1f8..6c46acf20442c2cc435829afa57e8383b493d6af 100644 --- a/tensorflow/contrib/data/__init__.py +++ b/tensorflow/contrib/data/__init__.py @@ -27,6 +27,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview. @@enumerate_dataset @@group_by_window @@ignore_errors +@@make_saveable_from_iterator @@read_batch_features @@unbatch @@rejection_resample @@ -49,13 +50,14 @@ from tensorflow.contrib.data.python.ops.dataset_ops import get_single_element from tensorflow.contrib.data.python.ops.enumerate_ops import enumerate_dataset from tensorflow.contrib.data.python.ops.error_ops import ignore_errors from tensorflow.contrib.data.python.ops.grouping import group_by_window +from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave +from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator from tensorflow.contrib.data.python.ops.readers import FixedLengthRecordDataset from tensorflow.contrib.data.python.ops.readers import read_batch_features from tensorflow.contrib.data.python.ops.readers import SqlDataset from tensorflow.contrib.data.python.ops.readers import TextLineDataset from tensorflow.contrib.data.python.ops.readers import TFRecordDataset from tensorflow.contrib.data.python.ops.resampling import rejection_resample -from tensorflow.contrib.data.python.ops.sloppy_ops import sloppy_interleave from tensorflow.python.data.ops.iterator_ops import Iterator # pylint: enable=unused-import diff --git a/tensorflow/contrib/data/kernels/BUILD b/tensorflow/contrib/data/kernels/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..4cb53741ebf8cd0db41b382c878bd2ccd1dcf7f1 --- /dev/null +++ b/tensorflow/contrib/data/kernels/BUILD @@ -0,0 +1,29 @@ +# Description: +# Contains kernels for datasets and iterators. +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +cc_library( + name = "prefetching_kernels", + srcs = ["prefetching_kernels.cc"], + deps = [ + "//tensorflow/core:framework_headers_lib", + "//third_party/eigen3", + "@protobuf_archive//:protobuf_headers", + ], + alwayslink = 1, +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), +) diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc new file mode 100644 index 0000000000000000000000000000000000000000..c9a3537c70c711290fb1111a1594e6dea3bc07a9 --- /dev/null +++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc @@ -0,0 +1,378 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/resource_op_kernel.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/util/device_name_utils.h" + +namespace tensorflow { + +struct BufferElement { + // The producer sets `status` if getting the input element fails. + Status status; + // The buffered data element. + std::vector value; +}; + +using FunctionBufferCallback = std::function; + +class FunctionBufferingResource : public ResourceBase { + public: + FunctionBufferingResource(FunctionLibraryRuntime* lib, + const NameAttrList& func, int64 buffer_size, + const string& source_device, + const string& target_device, + const std::vector& func_args, + int64 thread_pool_size) + : lib_(lib), + func_(func), + buffer_size_(buffer_size), + source_device_(source_device), + target_device_(target_device), + func_args_(func_args), + thread_pool_(new thread::ThreadPool(Env::Default(), ThreadOptions(), + "buffer_resource", thread_pool_size, + false /* low_latency_hint */)), + handle_(kInvalidHandle), + is_buffering_(false), + end_of_sequence_(false), + cancelled_(false) { + runner_ = [this](std::function c) { + thread_pool_->Schedule(std::move(c)); + }; + } + + ~FunctionBufferingResource() override { + Cancel(); + { + mutex_lock l(mu_); + while (is_buffering_) { + cond_var_.wait(l); + } + } + delete thread_pool_; + } + + string DebugString() override { + return strings::StrCat("FunctionBufferingResource. Size: ", buffer_size_, + "; target_device: ", target_device_); + } + + // Instantiates the function the first time it's called. After that it caches + // the handle. + Status Instantiate() LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + // Re-use existing handle if it's been set, effectively caching it. + if (handle_ != kInvalidHandle) { + return Status::OK(); + } + AttrValueMap attr_values = func_.attr(); + AttrValue v; + v.set_s(target_device_); + AddAttr("_target", v, &attr_values); + + return lib_->Instantiate(func_.name(), AttrSlice(&attr_values), &handle_); + } + + // Returns true if we've got to the end of the sequence and exhausted the + // buffer. + bool Finished() LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + return end_of_sequence_ && buffer_.empty(); + } + + // Cancels any buffering / prefetching going on. + void Cancel() LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + cancelled_ = true; + } + + // If the buffer has anything, runs `callback` on the first element in the + // buffer, else schedules the `callback` to be called. Requires `args` and + // `lib` in case more function calls need to be scheduled. + void MaybeGet(FunctionBufferCallback callback) LOCKS_EXCLUDED(mu_) { + bool start_buffering = false; + bool produced_output = false; + BufferElement buffer_element; + { + mutex_lock l(mu_); + if (!is_buffering_ && !end_of_sequence_) { + start_buffering = true; + } + if (!buffer_.empty()) { + produced_output = true; + std::swap(buffer_element, buffer_.front()); + buffer_.pop_front(); + } else { + produced_output = false; + requests_.push_back(std::move(callback)); + } + } + if (produced_output) { + callback(buffer_element); + } + if (start_buffering) { + FillBuffer(); + } + } + + private: + void FillBuffer() LOCKS_EXCLUDED(mu_) { + FunctionLibraryRuntime::Handle handle; + std::vector cancellation_callbacks; + std::vector cancellation_buffer_elements; + bool cancelled = false; + { + mutex_lock l(mu_); + handle = handle_; + if (cancelled_) { + cancelled = true; + // Run through and fulfill all pending requests, if possible. + while (!requests_.empty()) { + if (!buffer_.empty()) { + cancellation_buffer_elements.push_back(std::move(buffer_.front())); + buffer_.pop_front(); + cancellation_callbacks.push_back(std::move(requests_.front())); + requests_.pop_front(); + } else { + LOG(ERROR) << "Buffer ran out of elements and we couldn't satisfy: " + << requests_.size() << " requests"; + break; + } + } + is_buffering_ = false; + } else { + is_buffering_ = true; + } + } + if (cancelled) { + for (int i = 0; i < cancellation_callbacks.size(); ++i) { + cancellation_callbacks[i](cancellation_buffer_elements[i]); + } + // We only wait on cond_var_ in the destructor, so there would atmost be + // one waiter to notify. + cond_var_.notify_one(); + return; + } + FunctionLibraryRuntime::Options opts; + // Copied from CapturedFunction::generate_step_id(); + opts.step_id = -std::abs(static_cast(random::New64())); + opts.runner = &runner_; + opts.source_device = source_device_; + AllocatorAttributes arg_alloc_attr; + arg_alloc_attr.set_on_host(true); + opts.args_alloc_attrs.push_back(arg_alloc_attr); + if (opts.source_device != target_device_) { + opts.remote_execution = true; + } + opts.create_rendezvous = true; + auto* rets = new std::vector; + lib_->Run(opts, handle, func_args_, rets, + [this, rets](const Status& status) { + FunctionBufferCallback callback = nullptr; + BufferElement buffer_front; + bool restart_buffering = false; + { + mutex_lock l(mu_); + BufferElement buffer_element; + buffer_element.status = status; + if (!status.ok()) { + end_of_sequence_ = true; + is_buffering_ = false; + buffer_.push_back(std::move(buffer_element)); + return; + } + buffer_element.value.swap(*rets); + buffer_.push_back(std::move(buffer_element)); + if (!requests_.empty()) { + buffer_front = std::move(buffer_.front()); + buffer_.pop_front(); + callback = std::move(requests_.front()); + requests_.pop_front(); + } + if (buffer_.size() < buffer_size_) { + restart_buffering = true; + } else { + is_buffering_ = false; + } + } + if (callback != nullptr) { + callback(buffer_front); + } + if (restart_buffering) { + FillBuffer(); + } + }); + } + + mutex mu_; + FunctionLibraryRuntime* lib_; + NameAttrList func_; + const int64 buffer_size_; + const string source_device_; + const string target_device_; + const std::vector func_args_; + thread::ThreadPool* thread_pool_; + FunctionLibraryRuntime::Handle handle_ GUARDED_BY(mu_); + std::deque buffer_ GUARDED_BY(mu_); + std::deque requests_ GUARDED_BY(mu_); + std::function)> runner_ = nullptr; + bool is_buffering_ GUARDED_BY(mu_); + bool end_of_sequence_ GUARDED_BY(mu_); + bool cancelled_ GUARDED_BY(mu_); + condition_variable cond_var_; +}; + +class FunctionBufferResourceHandleOp : public OpKernel { + public: + explicit FunctionBufferResourceHandleOp(OpKernelConstruction* ctx) + : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("buffer_size", &buffer_size_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("container", &container_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("shared_name", &name_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("thread_pool_size", &thread_pool_size_)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor* string_arg; + OP_REQUIRES_OK(ctx, ctx->input("string_arg", &string_arg)); + std::vector func_args; + func_args.push_back(*string_arg); + + // Obtain and canonicalize target_device. + const Tensor* target_arg; + OP_REQUIRES_OK(ctx, ctx->input("target_device", &target_arg)); + const string& target_device = + DeviceNameUtils::CanonicalizeDeviceName(target_arg->scalar()()); + + FunctionLibraryRuntime* lib = ctx->function_library(); + OP_REQUIRES(ctx, lib != nullptr, + errors::Internal("No function library is provided.")); + + const string& source_device = ctx->device()->name(); + + ContainerInfo cinfo; + OP_REQUIRES_OK(ctx, cinfo.Init(ctx->resource_manager(), def())); + // Create the resource. + FunctionBufferingResource* buffer; + OP_REQUIRES_OK( + ctx, ctx->resource_manager()->LookupOrCreate( + cinfo.container(), cinfo.name(), &buffer, + [lib, &source_device, &target_device, func_args, + this](FunctionBufferingResource** ptr) { + *ptr = new FunctionBufferingResource( + lib, func_, buffer_size_, source_device, target_device, + func_args, thread_pool_size_); + return Status::OK(); + })); + OP_REQUIRES_OK(ctx, buffer->Instantiate()); + + OP_REQUIRES_OK(ctx, MakeResourceHandleToOutput( + ctx, 0, cinfo.container(), cinfo.name(), + MakeTypeIndex())); + } + + private: + NameAttrList func_; + int64 buffer_size_; + string container_; + string name_; + int64 thread_pool_size_; +}; + +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource") + .Device(DEVICE_CPU) + .HostMemory("resource") + .HostMemory("string_arg") + .HostMemory("target_device"), + FunctionBufferResourceHandleOp); +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource") + .Device(DEVICE_GPU) + .HostMemory("resource") + .HostMemory("string_arg") + .HostMemory("target_device"), + FunctionBufferResourceHandleOp); +#if TENSORFLOW_USE_SYCL +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource") + .Device(DEVICE_SYCL) + .HostMemory("resource") + .HostMemory("string_arg") + .HostMemory("target_device"), + FunctionBufferResourceHandleOp); +#endif // TENSORFLOW_USE_SYCL + +// Prefetches and fills up a buffer by calling a function that provides the +// elements to buffer. +class FunctionBufferingResourceGetNextOp : public AsyncOpKernel { + public: + explicit FunctionBufferingResourceGetNextOp(OpKernelConstruction* ctx) + : AsyncOpKernel(ctx) {} + + ~FunctionBufferingResourceGetNextOp() override {} + + void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override { + ResourceHandle handle; + OP_REQUIRES_OK_ASYNC( + ctx, HandleFromInput(ctx, "function_buffer_resource", &handle), done); + FunctionBufferingResource* buffer = nullptr; + OP_REQUIRES_OK_ASYNC( + ctx, LookupResource(ctx, handle, &buffer), + done); + core::ScopedUnref s(buffer); + + if (buffer->Finished()) { + ctx->SetStatus(errors::OutOfRange("end_of_sequence")); + done(); + return; + } + + FunctionBufferCallback callback = + [ctx, done](const BufferElement& buffer_element) { + Status s = buffer_element.status; + if (!s.ok()) { + ctx->SetStatus(s); + done(); + return; + } + for (size_t i = 0; i < buffer_element.value.size(); ++i) { + ctx->set_output(i, buffer_element.value[i]); + } + done(); + }; + buffer->MaybeGet(std::move(callback)); + } +}; + +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext") + .Device(DEVICE_CPU) + .HostMemory("function_buffer_resource"), + FunctionBufferingResourceGetNextOp); +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext") + .Device(DEVICE_GPU) + .HostMemory("function_buffer_resource"), + FunctionBufferingResourceGetNextOp); +#if TENSORFLOW_USE_SYCL +REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext") + .Device(DEVICE_SYCL) + .HostMemory("function_buffer_resource"), + FunctionBufferingResourceGetNextOp); +#endif // TENSORFLOW_USE_SYCL + +} // namespace tensorflow diff --git a/tensorflow/contrib/data/ops/prefetching_ops.cc b/tensorflow/contrib/data/ops/prefetching_ops.cc new file mode 100644 index 0000000000000000000000000000000000000000..23cb62b6f0dbfed15667dd00ae0039b33aa944d4 --- /dev/null +++ b/tensorflow/contrib/data/ops/prefetching_ops.cc @@ -0,0 +1,58 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" + +namespace tensorflow { + +REGISTER_OP("FunctionBufferingResource") + .Input("string_arg: string") + .Input("target_device: string") + .Output("resource: resource") + .Attr("shared_name: string") + .Attr("container: string") + .Attr("f: func") + .Attr("buffer_size: int") + .Attr("thread_pool_size: int") + .SetShapeFn(shape_inference::UnknownShape) + .Doc(R"doc( +Creates a resource that fills up a buffer by making function calls. + +string_arg: String argument to the function call. +target_device: Target device to execute the function on. +resource: Handle to the resource created. +f: Function to be executed. +buffer_size: Size of the buffer. +thread_pool_size: Size of the threadpool doing the prefetching. +container: If non-empty, this resource is placed in the given container. + Otherwise, a default container is used. +shared_name: If non-empty, this resource will be shared under the given name + across multiple sessions. +)doc"); + +REGISTER_OP("FunctionBufferingResourceGetNext") + .Input("function_buffer_resource: resource") + .Attr("output_types: list(type)") + .Output("output: output_types") + .SetShapeFn(shape_inference::UnknownShape) + .Doc(R"doc( +Gets the next element from a FunctionBufferingResource. + +function_buffer_resource: The FunctionBufferingResource handle. +output: A list of return values. +output_types: The type list for the return values. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD index c34c9dad9b5afb1f1232c8bff4c26770199ce7b6..424eb198522ce3d11152c2f8da6a2a5d82432cec 100644 --- a/tensorflow/contrib/data/python/kernel_tests/BUILD +++ b/tensorflow/contrib/data/python/kernel_tests/BUILD @@ -74,9 +74,12 @@ py_test( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", "//tensorflow/python:tensor_shape", + "//tensorflow/python:training", "//tensorflow/python/data/util:nest", "//third_party/py/numpy", ], @@ -93,6 +96,7 @@ py_test( ], deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -104,6 +108,7 @@ py_test( "//tensorflow/python:resource_variable_ops", "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", + "//tensorflow/python:training", "//tensorflow/python/data/util:nest", "//third_party/py/numpy", ], @@ -143,6 +148,29 @@ py_test( ], ) +py_test( + name = "interleave_dataset_op_test", + size = "small", + srcs = ["interleave_dataset_op_test.py"], + srcs_version = "PY2AND3", + tags = [ + "manual", # b/67958761 + ], + deps = [ + "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:math_ops", + "//tensorflow/python:script_ops", + "//tensorflow/python:training", + "//third_party/py/numpy", + ], +) + py_test( name = "iterator_ops_cluster_test", size = "small", @@ -185,6 +213,7 @@ py_test( "//tensorflow/python:function", "//tensorflow/python:functional_ops", "//tensorflow/python:gradients", + "//tensorflow/python:io_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:script_ops", @@ -217,6 +246,7 @@ py_test( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -224,6 +254,7 @@ py_test( "//tensorflow/python:data_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", "//tensorflow/python:functional_ops", "//tensorflow/python:io_ops", "//tensorflow/python:lookup_ops", @@ -231,6 +262,7 @@ py_test( "//tensorflow/python:random_ops", "//tensorflow/python:script_ops", "//tensorflow/python:string_ops", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//third_party/py/numpy", @@ -244,6 +276,7 @@ py_test( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -252,8 +285,11 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", + "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:tensor_shape", + "//tensorflow/python:training", "//tensorflow/python:variables", "//tensorflow/python/data/ops:iterator_ops", ], @@ -265,6 +301,7 @@ py_test( srcs = ["reader_dataset_ops_test.py"], srcs_version = "PY2AND3", deps = [ + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/contrib/data/python/ops:readers", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -274,9 +311,11 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", "//tensorflow/python:lib", "//tensorflow/python:parsing_ops", "//tensorflow/python:tensor_shape", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python/data/ops:iterator_ops", ], @@ -294,11 +333,8 @@ py_test( "//tensorflow/contrib/data/python/ops:transformation_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", - "//tensorflow/python:framework_ops", "//tensorflow/python:string_ops", - "//tensorflow/python:training", "//tensorflow/python:util", - "//tensorflow/python:variables", "//third_party/py/numpy", ], ) @@ -348,51 +384,50 @@ py_test( ) py_test( - name = "sloppy_transformation_dataset_op_test", + name = "sql_dataset_op_test", size = "small", - srcs = ["sloppy_transformation_dataset_op_test.py"], + srcs = ["sql_dataset_op_test.py"], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/contrib/data/python/ops:transformation_ops", + "//tensorflow/contrib/data/python/ops:readers", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:math_ops", - "//tensorflow/python:script_ops", - "//tensorflow/python:training", - "//third_party/py/numpy", ], ) py_test( - name = "sql_dataset_op_test", + name = "zip_dataset_op_test", size = "small", - srcs = ["sql_dataset_op_test.py"], + srcs = ["zip_dataset_op_test.py"], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/data/python/ops:readers", + "//tensorflow/contrib/data/python/ops:dataset_ops", + "//tensorflow/contrib/data/python/ops:iterator_ops", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:training", + "//tensorflow/python/data/util:nest", + "//third_party/py/numpy", ], ) py_test( - name = "zip_dataset_op_test", + name = "prefetching_ops_test", size = "small", - srcs = ["zip_dataset_op_test.py"], + srcs = ["prefetching_ops_test.py"], srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/data/python/ops:dataset_ops", - "//tensorflow/python:array_ops", + "//tensorflow/contrib/data/python/ops:prefetching_py", + "//tensorflow/core:protos_all_py", "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", - "//tensorflow/python:errors", - "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py index a77f3232ceb5bb34a3c35711d0d1cad13fbe2e0b..870352209a08e6bc08bcca227ba455ad1851e8bf 100644 --- a/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/concatenate_dataset_op_test.py @@ -17,13 +17,17 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import numpy as np from tensorflow.contrib.data.python.ops import dataset_ops +from tensorflow.contrib.data.python.ops import iterator_ops from tensorflow.python.data.util import nest from tensorflow.python.framework import errors +from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib class ConcatenateDatasetTest(test.TestCase): @@ -129,6 +133,140 @@ class ConcatenateDatasetTest(test.TestCase): with self.assertRaisesRegexp(TypeError, "have different types"): input_dataset.concatenate(dataset_to_concatenate) + def _iterator_checkpoint_prefix(self): + return os.path.join(self.get_temp_dir(), "iterator") + + def _build_graph(self, input_components, to_concatenate_components): + input_dataset = dataset_ops.Dataset.from_tensor_slices(input_components) + dataset_to_concatenate = dataset_ops.Dataset.from_tensor_slices( + to_concatenate_components) + iterator = input_dataset.concatenate( + dataset_to_concatenate).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + saveable = iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + # TODO(shivaniagrawal) : non-intuitive way, add support in mata_graph + for t in nest.flatten(get_next): + ops.add_to_collection("get_next", t) + return init_op, get_next + + def _testSaveRestoreUtility(self, start, break_range, stop): + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + input_components = (np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 4)) + to_concatenate_components = (np.tile( + np.array([[5], [6], [7], [8], [9]]), 20), np.tile( + np.array([[16], [17], [18], [19], [20]]), 15)) + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph(input_components, + to_concatenate_components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(start, break_range): + result = sess.run(get_next) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_range, stop): + result = sess.run(get_next) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testRestoreAtFirstDataset(self): + start = 0 + stop = 9 + break_range = 3 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreAtSecondDataset(self): + start = 0 + stop = 9 + break_range = 6 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreAtBetweenDatasets(self): + start = 0 + stop = 9 + break_range = 4 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreExhaustedIterator(self): + start = 0 + stop = 9 + break_range = 9 + self._testSaveRestoreUtility(start, break_range, stop) + + def testRestoreInModifiedGraph(self): + start = 0 + stop = 9 + break_range = 6 + path = self._iterator_checkpoint_prefix() + step = 0 + + input_components = (np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 4)) + to_concatenate_components = (np.tile( + np.array([[5], [6], [7], [8], [9]]), 20), np.tile( + np.array([[16], [17], [18], [19], [20]]), 15)) + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph(input_components, + to_concatenate_components) + saver = saver_lib.Saver(allow_empty=True) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(start, break_range): + result = sess.run(get_next) + if i < 4: + for component, result_component in zip(input_components, result): + self.assertAllEqual(component[i], result_component) + else: + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + saver.save(sess, path, step) + + new_to_concatenate_components = (np.array([[5], [6], [7], [8], [9]]), + np.array([[16], [17], [18], [19], [20]])) + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph(input_components, + new_to_concatenate_components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_range, stop): + result = sess.run(get_next) + for component, result_component in zip(to_concatenate_components, + result): + self.assertAllEqual(component[i - 4], result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py b/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py index a66714feda98d24778d9049b19455f28e4f76197..c3d6bfc097798530008f186cce68906b6af8fe47 100644 --- a/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/dataset_constructor_op_test.py @@ -17,12 +17,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os import threading import numpy as np from tensorflow.contrib.data.python.ops import batching from tensorflow.contrib.data.python.ops import dataset_ops +from tensorflow.contrib.data.python.ops import iterator_ops from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session from tensorflow.python.data.util import nest @@ -34,6 +36,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib class DatasetConstructorTest(test.TestCase): @@ -571,6 +574,136 @@ class DatasetConstructorTest(test.TestCase): new = batching._RestructuredDataset(dataset, new_types, new_shape_lists) # pylint: enable=protected-access + def _iterator_checkpoint_prefix(self): + return os.path.join(self.get_temp_dir(), "iterator") + + def _testSaveRestoreFromTensorsUtility(self, start, break_range, stop): + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + components = (np.array(1), np.array([1, 2, 3]), np.array(37.0)) + + with ops.Graph().as_default() as g: + iterator = ( + dataset_ops.Dataset.from_tensors(components) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + saveable = iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + for t in nest.flatten(get_next): + ops.add_to_collection("get_next", t) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(start, break_range): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component, result_component) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b", "c"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for _ in range(break_range, stop): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component, result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testRestoreFromTensors(self): + self._testSaveRestoreFromTensorsUtility(0, 0, 1) + + def testRestoreExhuatedIteratorFromTensors(self): + self._testSaveRestoreFromTensorsUtility(0, 1, 1) + + def _build_graph_tensor_slices(self, components): + iterator = dataset_ops.Dataset.from_tensor_slices( + components).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + saveable = iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + for t in nest.flatten(get_next): + ops.add_to_collection("get_next", t) + return init_op, get_next + + def _testSaveRestoreFromTensorSlicesUtility(self, start, break_range, stop): + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + components = (np.tile(np.array([[1], [2], [3], [4]]), 20), np.tile( + np.array([[12], [13], [14], [15]]), 22), + np.array([37.0, 38.0, 39.0, 40.0])) + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph_tensor_slices(components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(start, break_range): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component[i], result_component) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b", "c"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_range, stop): + result = sess.run(get_next) + for component, result_component in zip(components, result): + self.assertAllEqual(component[i], result_component) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testRestoreFromTensorSlices(self): + self._testSaveRestoreFromTensorSlicesUtility(0, 4, 2) + + def testRestoreExhaustedIteratorFromTensorSlices(self): + self._testSaveRestoreFromTensorSlicesUtility(0, 4, 4) + + def tesRestoreFromTensorSlicesWithDict(self): + + path = self._iterator_checkpoint_prefix() + step = 0 + meta_filename = path + "-%d.meta" % step + + components = {"foo": [1, 2, 3], "bar": [[4.0], [5.0], [6.0]]} + + with ops.Graph().as_default() as g: + init_op, get_next = self._build_graph_tensor_slices(components) + saver = saver_lib.Saver() + with self.test_session(graph=g) as sess: + sess.run(init_op) + for i in range(2): + results = sess.run(get_next) + self.assertEqual(components["foo"][i], results["foo"]) + self.assertEqual(components["bar"][i], results["bar"]) + saver.save(sess, path, step) + + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + with self.test_session(graph=g) as sess: + get_next = nest.pack_sequence_as(("a", "b"), + ops.get_collection("get_next")) + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(2, 3): + results = sess.run(get_next) + self.assertEqual(components["foo"][i], results["foo"]) + self.assertEqual(components["bar"][i], results["bar"]) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py similarity index 84% rename from tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py rename to tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py index 880e01dc069a70ac4ccbbbc18865f631ddea74d8..0aa9ea88de82b0851b0236d9412039d6573ab291 100644 --- a/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py @@ -25,7 +25,7 @@ import time from six.moves import zip_longest from tensorflow.contrib.data.python.ops import dataset_ops -from tensorflow.contrib.data.python.ops import sloppy_ops +from tensorflow.contrib.data.python.ops import interleave_ops from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.ops import array_ops @@ -34,12 +34,13 @@ from tensorflow.python.ops import script_ops from tensorflow.python.platform import test -class SloppyInterleaveDatasetTest(test.TestCase): +class ParallelInterleaveDatasetTest(test.TestCase): def setUp(self): self.input_values = array_ops.placeholder(dtypes.int64, shape=[None]) self.cycle_length = array_ops.placeholder(dtypes.int64, shape=[]) self.block_length = array_ops.placeholder(dtypes.int64, shape=[]) + self.sloppy = array_ops.placeholder(dtypes.bool, shape=[]) self.repeat_count = 2 @@ -69,9 +70,9 @@ class SloppyInterleaveDatasetTest(test.TestCase): self.dataset = (dataset_ops.Dataset.from_tensor_slices(self.input_values) .repeat(self.repeat_count).apply( - sloppy_ops.sloppy_interleave( + interleave_ops.parallel_interleave( interleave_fn, self.cycle_length, - self.block_length))) + self.block_length, self.sloppy))) self.iterator = self.dataset.make_initializable_iterator() self.init_op = self.iterator.initializer self.next_element = self.iterator.get_next() @@ -161,7 +162,7 @@ class SloppyInterleaveDatasetTest(test.TestCase): for i in range(4, 7): self.write_coordination_events[i].set() - def testSingleThreaded(self): + def _testSingleThreaded(self, sloppy=False): # cycle_length=1,block_length=1 acts like `Dataset.interleave()` and # `Dataset.flat_map()` and is single-threaded. No synchronization required. with self.test_session() as sess: @@ -171,7 +172,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 1, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy }) for expected_element in self._interleave( @@ -182,7 +184,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContention(self): + def testSingleThreaded(self): + self._testSingleThreaded() + + def testSingleThreadedSloppy(self): + self._testSingleThreaded(sloppy=True) + + def _testTwoThreadsNoContention(self, sloppy=False): # num_threads > 1. # Explicit coordination should result in `Dataset.interleave()` behavior with self.test_session() as sess: @@ -193,7 +201,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -211,11 +220,20 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContentionWithRaces(self): + def testTwoThreadsNoContention(self): + self._testTwoThreadsNoContention() + + def testTwoThreadsNoContentionSloppy(self): + self._testTwoThreadsNoContention(sloppy=True) + + def _testTwoThreadsNoContentionWithRaces(self, sloppy=False): """Tests where all the workers race in producing elements. Note: this is in contrast with the prevous test which carefully sequences the execution of the map functions. + + Args: + sloppy: Whether to be sloppy or not. """ with self.test_session() as sess: self._clear_coordination_events() @@ -225,7 +243,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy, }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -247,7 +266,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContentionBlockLength(self): + def testTwoThreadsNoContentionWithRaces(self): + self._testTwoThreadsNoContentionWithRaces() + + def testTwoThreadsNoContentionWithRacesSloppy(self): + self._testTwoThreadsNoContentionWithRaces(sloppy=True) + + def _testTwoThreadsNoContentionBlockLength(self, sloppy=False): # num_threads > 1. # Explicit coordination should result in `Dataset.interleave()` behavior with self.test_session() as sess: @@ -258,7 +283,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 2 + self.block_length: 2, + self.sloppy: sloppy }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -276,11 +302,21 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testTwoThreadsNoContentionWithRacesAndBlocking(self): + def testTwoThreadsNoContentionBlockLength(self): + self._testTwoThreadsNoContentionBlockLength() + + def testTwoThreadsNoContentionBlockLengthSloppy(self): + self._testTwoThreadsNoContentionBlockLength(sloppy=True) + + def _testTwoThreadsNoContentionWithRacesAndBlocking(self, sloppy=False): """Tests where all the workers race in producing elements. Note: this is in contrast with the prevous test which carefully sequences the execution of the map functions. + + + Args: + sloppy: Whether to be sloppy or not. """ with self.test_session() as sess: self._clear_coordination_events() @@ -290,7 +326,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 2 + self.block_length: 2, + self.sloppy: sloppy }) for i, expected_element in enumerate( self._interleave([[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 2, @@ -312,7 +349,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testEmptyInput(self): + def testTwoThreadsNoContentionWithRacesAndBlocking(self): + self._testTwoThreadsNoContentionWithRacesAndBlocking() + + def testTwoThreadsNoContentionWithRacesAndBlockingSloppy(self): + self._testTwoThreadsNoContentionWithRacesAndBlocking(sloppy=True) + + def _testEmptyInput(self, sloppy=False): with self.test_session() as sess: # Empty input. self._clear_coordination_events() @@ -321,12 +364,19 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [], self.cycle_length: 2, - self.block_length: 3 + self.block_length: 3, + self.sloppy: sloppy }) with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testNonEmptyInputIntoEmptyOutputs(self): + def testEmptyInput(self): + self._testEmptyInput() + + def testEmptyInputSloppy(self): + self._testEmptyInput(sloppy=True) + + def _testNonEmptyInputIntoEmptyOutputs(self, sloppy=False): # Non-empty input leading to empty output. with self.test_session() as sess: self._clear_coordination_events() @@ -335,12 +385,19 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [0, 0, 0], self.cycle_length: 2, - self.block_length: 3 + self.block_length: 3, + self.sloppy: sloppy }) with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testPartiallyEmptyOutputs(self): + def testNonEmptyInputIntoEmptyOutputs(self): + self._testNonEmptyInputIntoEmptyOutputs() + + def testNonEmptyInputIntoEmptyOutputsSloppy(self): + self._testNonEmptyInputIntoEmptyOutputs(sloppy=True) + + def _testPartiallyEmptyOutputs(self, sloppy=False): # Mixture of non-empty and empty interleaved datasets. with self.test_session() as sess: self._clear_coordination_events() @@ -350,7 +407,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 0, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: sloppy, }) for i, expected_element in enumerate( self._interleave([[4] * 4, [], [6] * 6] * self.repeat_count, 2, 1)): @@ -367,7 +425,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testDelayedOutput(self): + def testPartiallyEmptyOutputs(self): + self._testPartiallyEmptyOutputs() + + def testPartiallyEmptyOutputsSloppy(self): + self._testPartiallyEmptyOutputs(sloppy=True) + + def testDelayedOutputSloppy(self): # Explicitly control the sequence of events to ensure we correctly avoid # head-of-line blocking. with self.test_session() as sess: @@ -377,7 +441,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 1 + self.block_length: 1, + self.sloppy: True, }) mis_ordering = [ @@ -391,7 +456,7 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testBlockLengthWithContention(self): + def testBlockLengthWithContentionSloppy(self): with self.test_session() as sess: self._clear_coordination_events() done_first_event = False @@ -400,7 +465,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 2, - self.block_length: 3 + self.block_length: 3, + self.sloppy: True }) # Test against a generating sequence that differs from the uncontended # case, in order to prove sloppy correctness. @@ -422,7 +488,7 @@ class SloppyInterleaveDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(self.next_element) - def testEarlyExit(self): + def _testEarlyExit(self, sloppy=False): # Exiting without consuming all input should not block with self.test_session() as sess: self._clear_coordination_events() @@ -431,7 +497,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): feed_dict={ self.input_values: [4, 5, 6], self.cycle_length: 3, - self.block_length: 2 + self.block_length: 2, + self.sloppy: sloppy }) for i in range(4, 7): self.write_coordination_events[i].set() @@ -445,7 +512,13 @@ class SloppyInterleaveDatasetTest(test.TestCase): self.read_coordination_events[i].acquire() self.write_coordination_events[i].set() - def testTooManyReaders(self): + def testEarlyExit(self): + self._testEarlyExit() + + def testEarlyExitSloppy(self): + self._testEarlyExit(sloppy=True) + + def _testTooManyReaders(self, sloppy=False): def interleave_fn(x): dataset = dataset_ops.Dataset.from_tensors(x) @@ -455,8 +528,8 @@ class SloppyInterleaveDatasetTest(test.TestCase): dataset = dataset_ops.Dataset.from_tensor_slices([4, 5, 6]) dataset = dataset.repeat(self.repeat_count) dataset = dataset.apply( - sloppy_ops.sloppy_interleave(interleave_fn, cycle_length=16, - block_length=2)) + interleave_ops.parallel_interleave( + interleave_fn, cycle_length=16, block_length=2, sloppy=sloppy)) iterator = dataset.make_one_shot_iterator() with self.test_session() as sess: @@ -468,6 +541,11 @@ class SloppyInterleaveDatasetTest(test.TestCase): [[4] * 4, [5] * 5, [6] * 6] * self.repeat_count, 1, 2) self.assertItemsEqual(output_values, expected_values) + def testTooManyReaders(self): + self._testTooManyReaders() + + def testTooManyReadersSloppy(self): + self._testTooManyReaders(sloppy=True) if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py index 20f6d6ba34f49fa99d42961a6aa68ffed6b4f657..bda9a2a4a37e9c3d35ff99041d1150ffc43f4c43 100644 --- a/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/iterator_ops_test.py @@ -35,6 +35,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import io_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import script_ops @@ -538,9 +539,23 @@ class IteratorTest(test.TestCase): def testIncorrectIteratorRestore(self): - def _iterator_checkpoint_prefix(): + def _path(): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + _path(), parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_range_dataset_graph(): start = 1 stop = 10 @@ -548,22 +563,18 @@ class IteratorTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = _iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op def _build_reader_dataset_graph(): filenames = ["test"] # Does not exist but we don't care in this test. - path = _iterator_checkpoint_prefix() iterator = readers.FixedLengthRecordDataset( filenames, 1, 0, 0).make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op # Saving iterator for RangeDataset graph. diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py new file mode 100644 index 0000000000000000000000000000000000000000..539c6f215536f50a0b56f173a9240542faa2e643 --- /dev/null +++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py @@ -0,0 +1,108 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for prefetching_ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import itertools +import threading + +from tensorflow.contrib.data.python.ops import prefetching_ops +from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.data.ops import iterator_ops +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.platform import test + + +class StagingAreaOpsTest(test.TestCase): + + def setUp(self): + self._event = threading.Event() + + def _prefetch_fn_helper(self, buffer_name, device0, device1): + worker_config = config_pb2.ConfigProto() + worker_config.device_count["CPU"] = 2 + + def gen(): + for i in itertools.count(start=1, step=1): + yield [i + 0.0] + if i == 6: + self._event.set() + + with ops.device(device0): + dataset_3 = dataset_ops.Dataset.from_generator(gen, (dtypes.float32)) + iterator_3 = dataset_3.make_one_shot_iterator() + iterator_3_handle = iterator_3.string_handle() + + @function.Defun(dtypes.string) + def _remote_fn(h): + remote_iterator = iterator_ops.Iterator.from_string_handle( + h, dataset_3.output_types, dataset_3.output_shapes) + return remote_iterator.get_next() + + target = constant_op.constant(device0) + with ops.device(device1): + buffer_resource_handle = prefetching_ops.function_buffering_resource( + f=_remote_fn, + target_device=target, + string_arg=iterator_3_handle, + buffer_size=3, + thread_pool_size=2, + shared_name=buffer_name) + + with ops.device(device1): + prefetch_op = prefetching_ops.function_buffering_resource_get_next( + function_buffer_resource=buffer_resource_handle, + output_types=[dtypes.float32]) + + with self.test_session(config=worker_config) as sess: + elem = sess.run(prefetch_op) + self.assertEqual(elem, [1.0]) + elem = sess.run(prefetch_op) + self.assertEqual(elem, [2.0]) + elem = sess.run(prefetch_op) + self.assertEqual(elem, [3.0]) + elem = sess.run(prefetch_op) + self.assertEqual(elem, [4.0]) + self._event.wait() + elem = sess.run(prefetch_op) + self.assertEqual(elem, [5.0]) + + def testSameDeviceCPU(self): + self._prefetch_fn_helper("same_device_cpu", + "/job:localhost/replica:0/task:0/cpu:0", + "/job:localhost/replica:0/task:0/cpu:0") + + def testDifferentDeviceCPU(self): + self._prefetch_fn_helper("diff_device_cpu", + "/job:localhost/replica:0/task:0/cpu:0", + "/job:localhost/replica:0/task:0/cpu:1") + + def testDifferentDeviceCPUGPU(self): + if not test_util.is_gpu_available(): + self.skipTest("No GPU available") + + self._prefetch_fn_helper("cpu_gpu", "/job:localhost/replica:0/task:0/cpu:0", + "/job:localhost/replica:0/task:0/gpu:0") + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py index c8a0072809c2eac30e255d29ecaee5a324449045..f59ac760dc83a504e563f055b91f1002cb0c80fc 100644 --- a/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/range_dataset_op_test.py @@ -21,6 +21,7 @@ import os from tensorflow.contrib.data.python.ops import dataset_ops from tensorflow.contrib.data.python.ops import enumerate_ops +from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops from tensorflow.python.data.ops import iterator_ops from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -29,9 +30,12 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops +from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib class RangeDatasetTest(test.TestCase): @@ -193,6 +197,21 @@ class RangeDatasetTest(test.TestCase): def _iterator_checkpoint_prefix(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_prefix(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def testSaveRestore(self): def _build_graph(start, stop): @@ -200,10 +219,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -244,16 +261,146 @@ class RangeDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(get_next) + def testSaveRestoreUsingSaverFromMetaGraph(self): + + def _build_graph(start, stop): + iterator = dataset_ops.Dataset.range(start, + stop).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + ops.add_to_collection("iterator_ops", init_op) + ops.add_to_collection("iterator_ops", get_next) + saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator) + # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection + # so that it can be automatically picked up by the Saver. + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj) + saver = saver_lib.Saver() + return init_op, get_next, saver + + start = 2 + stop = 10 + break_point = 5 + path = self._iterator_checkpoint_prefix() + meta_filename = path + ".meta" + + # Execute input pipeline for a few steps and save iterator state. + with ops.Graph().as_default() as g: + init_op, get_next, saver = _build_graph(start, stop) + with self.test_session(graph=g) as sess: + sess.run(variables.global_variables_initializer()) + sess.run(init_op) + for i in range(start, break_point): + self.assertEqual(i, sess.run(get_next)) + saver.save(sess, path) + + # Build the saver from the MetaGraph using import_meta_graph and + # check that the iterator state is restored. + with ops.Graph().as_default() as g: + saver = saver_lib.import_meta_graph(meta_filename) + init_op, get_next = ops.get_collection("iterator_ops") + with self.test_session(graph=g) as sess: + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_point, stop): + self.assertEqual(i, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testSaveRestoreUsingBuiltSaver(self): + + def _build_graph(start, stop): + iterator = dataset_ops.Dataset.range(start, + stop).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + ops.add_to_collection("iterator_ops", init_op) + ops.add_to_collection("iterator_ops", get_next) + # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection + # so that it can be automatically picked up by the Saver. + saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj) + saver = saver_lib.Saver() + return init_op, get_next, saver + + start = 2 + stop = 10 + stop_new = 15 + break_point = 5 + path = self._iterator_checkpoint_prefix() + + # Execute input pipeline for a few steps and save iterator state. + with ops.Graph().as_default() as g: + init_op, get_next, saver = _build_graph(start, stop) + with self.test_session(graph=g) as sess: + sess.run(variables.global_variables_initializer()) + sess.run(init_op) + for i in range(start, break_point): + self.assertEqual(i, sess.run(get_next)) + saver.save(sess, path) + + # Manually build a modified Graph and Saver instead of importing + # MetaGraph and verify that original iterator state gets restored. + with ops.Graph().as_default() as g: + init_op, get_next, saver = _build_graph(start, stop_new) + with self.test_session(graph=g) as sess: + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + for i in range(break_point, stop): + self.assertEqual(i, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testSaveRestoreUsingSaverThenInit(self): + + def _build_graph(start, stop): + iterator = dataset_ops.Dataset.range(start, + stop).make_initializable_iterator() + init_op = iterator.initializer + get_next = iterator.get_next() + ops.add_to_collection("iterator_ops", init_op) + ops.add_to_collection("iterator_ops", get_next) + # Add the SaveableObject to the `SAVEABLE_OBJECTS` collection + # so that it can be automatically picked up by the Saver. + saveable_obj = contrib_iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable_obj) + saver = saver_lib.Saver() + return init_op, get_next, saver + + start = 2 + stop = 10 + stop_new = 15 + break_point = 5 + path = self._iterator_checkpoint_prefix() + + # Execute input pipeline for a few steps and save iterator state. + with ops.Graph().as_default() as g: + init_op, get_next, saver = _build_graph(start, stop) + with self.test_session(graph=g) as sess: + sess.run(variables.global_variables_initializer()) + sess.run(init_op) + for i in range(start, break_point): + self.assertEqual(i, sess.run(get_next)) + saver.save(sess, path) + + # Restore iterator state call and then call init_op for the iterator and + # verify that the new iterator hides the restored iterator. + with ops.Graph().as_default() as g: + init_op, get_next, saver = _build_graph(start, stop_new) + with self.test_session(graph=g) as sess: + saver.restore(sess, saver_lib.latest_checkpoint(self.get_temp_dir())) + sess.run(init_op) + for i in range(start, stop_new): + self.assertEqual(i, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + def testRestoreWithoutBuildingDatasetGraph(self): - def _build_graph(start, stop, num_epochs, path): + def _build_graph(start, stop, num_epochs): dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -262,10 +409,8 @@ class RangeDatasetTest(test.TestCase): num_epochs = 5 break_point = 5 break_epoch = 3 - path = self._iterator_checkpoint_prefix() with ops.Graph().as_default() as g: - init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs, - path) + init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs) with self.test_session(graph=g) as sess: sess.run(variables.global_variables_initializer()) sess.run(init_op) @@ -282,8 +427,7 @@ class RangeDatasetTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + restore_op = self._restore_op(iterator._iterator_resource) get_next = iterator.get_next() with self.test_session(graph=g) as sess: sess.run(restore_op) @@ -302,10 +446,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -343,10 +485,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -379,10 +519,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -424,10 +562,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -471,10 +607,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py index c9f88f3dfc9a062ccd0bcabe7eadf18c98191c1d..3ae8f71d77fa6ecf08e42bedac702b8f75eec309 100644 --- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py @@ -21,6 +21,7 @@ import gzip import os import zlib +from tensorflow.contrib.data.python.ops import iterator_ops as contrib_iterator_ops from tensorflow.contrib.data.python.ops import readers from tensorflow.core.example import example_pb2 from tensorflow.core.example import feature_pb2 @@ -33,8 +34,10 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.lib.io import python_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test +from tensorflow.python.training import saver as saver_lib from tensorflow.python.util import compat @@ -162,6 +165,277 @@ class TextLineDatasetTest(test.TestCase): with self.assertRaises(errors.OutOfRangeError): sess.run(iterator.get_next()) + def _ckpt_path(self): + return os.path.join(self.get_temp_dir(), "iterator") + + def _latest_ckpt(self): + return saver_lib.latest_checkpoint(self.get_temp_dir()) + + def _save(self, saver, sess): + saver.save(sess, self._ckpt_path()) + + def _restore(self, saver, sess): + saver.restore(sess, self._latest_ckpt()) + + def _import_meta_graph(self): + meta_file_path = self._ckpt_path() + ".meta" + return saver_lib.import_meta_graph(meta_file_path) + + def _build_graph(self, + test_filenames, + compression_type=None, + build_saveable=True): + ds = readers.TextLineDataset( + test_filenames, compression_type=compression_type, buffer_size=10) + iterator = ds.make_initializable_iterator() + if build_saveable: + saveable = contrib_iterator_ops.make_saveable_from_iterator(iterator) + ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable) + init_op = iterator.initializer + get_next = iterator.get_next() + ops.add_to_collection("iterator_ops", init_op) + ops.add_to_collection("iterator_ops", get_next) + saver = saver_lib.Saver(allow_empty=True) + return init_op, get_next, saver + + def _testReadWithBreaks(self, breaks, num_files=5, lines_per_file=5): + """Tests reading from input pipeline with regular breaks. + + At each break point the iterator state gets saved using Saver and reloaded + in a new Graph and session. + + Args: + breaks: List of counts of records after reading which iterator state is + checkpointed. Must to in non-decreasing order. + num_files: Total number of files. + lines_per_file: Total number of lines per file. + """ + compression_types = [None, "GZIP", "ZLIB"] + for compression_type in compression_types: + test_filenames = self._createFiles( + num_files, + lines_per_file, + crlf=True, + compression_type=compression_type) + + # Collect ground truth. + total_records = num_files * lines_per_file + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type=compression_type) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(total_records): + expected_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + # Simulate run with breaks. + actual_records = [] + next_record_index = 0 + load_from_ckpt = False + breaks.append(total_records) + for break_index in breaks: + with ops.Graph().as_default() as g: + if not load_from_ckpt: + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type=compression_type) + else: + saver = self._import_meta_graph() + init_op, get_next = ops.get_collection("iterator_ops") + + with self.test_session(graph=g) as sess: + if not load_from_ckpt: + sess.run(init_op) + else: + self._restore(saver, sess) + while next_record_index != break_index: + actual_records.append(sess.run(get_next)) + next_record_index += 1 + if break_index == total_records: + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self._save(saver, sess) + load_from_ckpt = True + self.assertEqual(actual_records, expected_records) + + def testSaveAtFileBoundary(self): + self._testReadWithBreaks([10]) + + def testSaveWithinFile(self): + self._testReadWithBreaks([12]) + + def testSaveUnusedIterator(self): + self._testReadWithBreaks([0]) + + def testSaveRestoreIdempotence(self): + # Attempt to save an iterator immediately after it has been + # restored. + self._testReadWithBreaks([0, 0]) + self._testReadWithBreaks([10, 10]) + self._testReadWithBreaks([12, 12]) + + def testMultipleBreaks(self): + self._testReadWithBreaks([0, 4, 20]) + + def testRestoreExhaustedIterator(self): + num_files = 2 + lines_per_file = 5 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(num_files * lines_per_file): + sess.run(get_next) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self._save(saver, sess) + + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + saver = self._import_meta_graph() + self._restore(saver, sess) + _, get_next = ops.get_collection("iterator_ops") + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testInitThenRestore(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + sess.run(get_next) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + saver = self._import_meta_graph() + init_op, get_next = ops.get_collection("iterator_ops") + sess.run(init_op) + self._restore(saver, sess) + for _ in range(total_records - break_record): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + + def testRestoreInModifiedGraph(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + sess.run(get_next) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type="GZIP") + self._restore(saver, sess) + for _ in range(total_records - break_record): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + + def testRestoreInModifiedGraphThenInit(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + expected_records.append(sess.run(get_next)) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + # Test that calling the init_op overrides the restored iterator. The + # iterator for the old graph was build to read uncompressed files and + # would fail when trying to read the new files. + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + test_filenames = self._createFiles( + num_files, lines_per_file, crlf=True, compression_type="GZIP") + init_op, get_next, saver = self._build_graph( + test_filenames, compression_type="GZIP") + self._restore(saver, sess) + sess.run(init_op) + for _ in range(total_records): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + + def testDoNotRestoreIterator(self): + num_files = 5 + lines_per_file = 5 + total_records = num_files * lines_per_file + break_record = 8 + test_filenames = self._createFiles(num_files, lines_per_file, crlf=True) + + expected_records = [] + with ops.Graph().as_default() as g: + init_op, get_next, saver = self._build_graph(test_filenames) + with self.test_session(graph=g) as sess: + sess.run(init_op) + for _ in range(break_record): + expected_records.append(sess.run(get_next)) + self._save(saver, sess) + for _ in range(total_records - break_record): + expected_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + actual_records = [] + with ops.Graph().as_default() as g: + with self.test_session(graph=g) as sess: + init_op, get_next, saver = self._build_graph( + test_filenames, build_saveable=False) + self._restore(saver, sess) + with self.assertRaises(errors.FailedPreconditionError): + sess.run(get_next) + sess.run(init_op) + for _ in range(total_records): + actual_records.append(sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + self.assertEqual(actual_records, expected_records) + class FixedLengthRecordReaderTest(test.TestCase): @@ -276,18 +550,31 @@ class FixedLengthRecordReaderTest(test.TestCase): def _iterator_checkpoint_path(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_path(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_iterator_graph(self, num_epochs): filenames = self._createFiles() - path = self._iterator_checkpoint_path() dataset = (readers.FixedLengthRecordDataset( filenames, self._record_bytes, self._header_bytes, self._footer_bytes) .repeat(num_epochs)) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op def _restore_iterator(self): @@ -295,8 +582,7 @@ class FixedLengthRecordReaderTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) get_next = iterator.get_next() - restore_op = gen_dataset_ops.restore_iterator( - iterator._iterator_resource, self._iterator_checkpoint_path()) + restore_op = self._restore_op(iterator._iterator_resource) return restore_op, get_next def testSaveRestore(self): diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD index 2a9b41d6df0b447d64dc6cf28961e08cab5f367f..1b81cf5be9190ffab646192fb9a72fd3da7deee1 100644 --- a/tensorflow/contrib/data/python/ops/BUILD +++ b/tensorflow/contrib/data/python/ops/BUILD @@ -4,6 +4,13 @@ licenses(["notice"]) # Apache 2.0 exports_files(["LICENSE"]) +load( + "//tensorflow:tensorflow.bzl", + "tf_gen_op_wrapper_py", + "tf_kernel_library", +) +load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") + py_library( name = "dataset_ops", srcs = [ @@ -12,14 +19,25 @@ py_library( srcs_version = "PY2AND3", deps = [ ":transformation_ops", - "//tensorflow/python:dtypes", - "//tensorflow/python:script_ops", - "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/util:nest", ], ) +py_library( + name = "iterator_ops", + srcs = [ + "iterator_ops.py", + ], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:dataset_ops_gen", + "//tensorflow/python:framework_ops", + "//tensorflow/python:training", + ], +) + py_library( name = "readers", srcs = [ @@ -35,6 +53,7 @@ py_library( "//tensorflow/python:platform", "//tensorflow/python:sparse_tensor", "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/ops:readers", "//tensorflow/python/data/util:nest", @@ -48,9 +67,9 @@ py_library( "enumerate_ops.py", "error_ops.py", "grouping.py", + "interleave_ops.py", "resampling.py", "scan_ops.py", - "sloppy_ops.py", ], srcs_version = "PY2AND3", deps = [ @@ -63,7 +82,6 @@ py_library( "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", - "//tensorflow/python:resource_variable_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", "//tensorflow/python/data/ops:dataset_ops", @@ -72,6 +90,44 @@ py_library( ], ) +tf_gen_op_wrapper_py( + name = "prefetching_ops", + out = "gen_prefetching_ops.py", + deps = ["//tensorflow/contrib/data:prefetching_ops_op_lib"], +) + +tf_kernel_library( + name = "prefetching_ops_kernels", + deps = [ + "//tensorflow/contrib/data/kernels:prefetching_kernels", + "//tensorflow/core:framework", + ], + alwayslink = 1, +) + +tf_custom_op_py_library( + name = "prefetching_py", + srcs = ["prefetching_ops.py"], + dso = ["//tensorflow/contrib/data:_prefetching_ops.so"], + kernels = [ + ":prefetching_ops_kernels", + "//tensorflow/contrib/data:prefetching_ops_op_lib", + ], + srcs_version = "PY2AND3", + deps = [ + ":prefetching_ops", + "//tensorflow/contrib/util:util_py", + "//tensorflow/python:client_testlib", + "//tensorflow/python:errors", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform", + "//tensorflow/python:state_ops", + "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/data/python/ops/dataset_ops.py b/tensorflow/contrib/data/python/ops/dataset_ops.py index fe1d50db33a994103695e1f418413ec6532de10a..45d6dbe7438957029b4d6b71e181cb1fc3596ecb 100644 --- a/tensorflow/contrib/data/python/ops/dataset_ops.py +++ b/tensorflow/contrib/data/python/ops/dataset_ops.py @@ -24,11 +24,8 @@ from tensorflow.contrib.data.python.ops import grouping from tensorflow.python.data.ops import dataset_ops from tensorflow.python.data.util import nest -from tensorflow.python.framework import dtypes -from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gen_io_ops -from tensorflow.python.ops import script_ops from tensorflow.python.util import deprecation @@ -139,124 +136,8 @@ class Dataset(dataset_ops.Dataset): Returns: A `Dataset`. """ - if not callable(generator): - raise TypeError("`generator` must be callable.") - if output_shapes is None: - output_shapes = nest.map_structure( - lambda _: tensor_shape.TensorShape(None), output_types) - else: - output_shapes = nest.map_structure_up_to( - output_types, tensor_shape.as_shape, output_shapes) - - flattened_types = nest.flatten(output_types) - flattened_shapes = nest.flatten(output_shapes) - - generator_state = dataset_ops.Dataset._GeneratorState(generator) - - def get_iterator_id_map_fn(unused_dummy): - """Creates a unique `iterator_id` for each pass over the dataset. - - The "iterator_id" disambiguates between multiple concurrently - existing iterators. - - Args: - unused_dummy: Ignored value. - - Returns: - A `tf.int64` tensor whose value uniquely identifies an iterator in - `generator_state`. - """ - return script_ops.py_func( - generator_state.get_next_id, [], dtypes.int64, stateful=True) - - def generator_map_fn(iterator_id_t): - """Generates the next element from iterator with ID `iterator_id_t`. - - We map this function across an infinite repetition of the - `iterator_id_t`, and raise `StopIteration` to terminate the iteration. - - Args: - iterator_id_t: A `tf.int64` tensor whose value uniquely identifies - the iterator in `generator_state` from which to generate an element. - - Returns: - A nested structure of tensors representing an element from the iterator. - """ - - def generator_py_func(iterator_id): - """A `py_func` that will be called to invoke the iterator.""" - try: - values = next(generator_state.get_iterator(iterator_id)) - except StopIteration: - generator_state.iterator_completed(iterator_id) - raise StopIteration("Iteration finished.") - - # Use the same _convert function from the py_func() implementation to - # convert the returned values to arrays early, so that we can inspect - # their values. - # pylint: disable=protected-access - ret_arrays = [ - script_ops.FuncRegistry._convert(ret, dtype=dtype.as_numpy_dtype) - for ret, dtype in zip(nest.flatten_up_to(output_types, values), - flattened_types) - ] - # pylint: enable=protected-access - - # Additional type and shape checking to ensure that the components - # of the generated element match the `output_types` and `output_shapes` - # arguments. - for (ret_array, expected_dtype, expected_shape) in zip( - ret_arrays, flattened_types, flattened_shapes): - if ret_array.dtype != expected_dtype.as_numpy_dtype: - raise TypeError( - "`generator` yielded an element of type %s where an element " - "of type %s was expected." % (ret_array.dtype, - expected_dtype.as_numpy_dtype)) - if not expected_shape.is_compatible_with(ret_array.shape): - raise ValueError( - "`generator` yielded an element of shape %s where an element " - "of shape %s was expected." % (ret_array.shape, expected_shape)) - - return ret_arrays - - flat_values = script_ops.py_func( - generator_py_func, [iterator_id_t], flattened_types, stateful=True) - - # The `py_func()` op drops the inferred shapes, so we add them back in - # here. - if output_shapes is not None: - for ret_t, shape in zip(flat_values, flattened_shapes): - ret_t.set_shape(shape) - - return nest.pack_sequence_as(output_types, flat_values) - - # This function associates each traversal of `generator` with a unique - # iterator ID. - def flat_map_fn(iterator_id_t): - # First, generate an infinite dataset containing the iterator ID repeated - # forever. - repeated_id = Dataset.from_tensors(iterator_id_t).repeat(None) - - # The `generator_map_fn` gets the next element from the iterator with the - # relevant ID, and raises StopIteration when that iterator contains no - # more elements. - return repeated_id.map(generator_map_fn) - - # A single-element dataset that, each time it is evaluated, contains a - # freshly-generated and unique (for the returned dataset) int64 - # ID that will be used to identify the appropriate Python state, which - # is encapsulated in `generator_state`, and captured in - # `get_iterator_id_map_fn`. - dummy = 0 - id_dataset = Dataset.from_tensors(dummy).map(get_iterator_id_map_fn) - - # A dataset that contains all of the elements generated by a - # single iterator created from `generator`, identified by the - # iterator ID contained in `id_dataset`. Lifting the iteration - # into a flat_map here enables multiple repetitions and/or nested - # versions of the returned dataset to be created, because it forces - # the generation of a new ID for each version. - return id_dataset.flat_map(flat_map_fn) + return Dataset(dataset_ops.Dataset.from_generator( + generator, output_types, output_shapes)) @staticmethod @deprecation.deprecated(None, "Use `tf.data.Dataset.range()`.") diff --git a/tensorflow/contrib/data/python/ops/sloppy_ops.py b/tensorflow/contrib/data/python/ops/interleave_ops.py similarity index 67% rename from tensorflow/contrib/data/python/ops/sloppy_ops.py rename to tensorflow/contrib/data/python/ops/interleave_ops.py index 4f3da4320cd7d550c5d93db7534ad9950401a8c6..74a919c1fff62cfa79b0877a3d081077ca6776f0 100644 --- a/tensorflow/contrib/data/python/ops/sloppy_ops.py +++ b/tensorflow/contrib/data/python/ops/interleave_ops.py @@ -23,14 +23,16 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.util import deprecation -class SloppyInterleaveDataset(dataset_ops.Dataset): +class ParallelInterleaveDataset(dataset_ops.Dataset): """A `Dataset` that maps a function over its input and flattens the result.""" - def __init__(self, input_dataset, map_func, cycle_length, block_length): - """See `tf.contrib.data.sloppy_interleave()` for details.""" - super(SloppyInterleaveDataset, self).__init__() + def __init__(self, input_dataset, map_func, cycle_length, block_length, + sloppy): + """See `tf.contrib.data.parallel_interleave()` for details.""" + super(ParallelInterleaveDataset, self).__init__() self._input_dataset = input_dataset @function.Defun(*nest.flatten(input_dataset.output_types)) @@ -62,13 +64,16 @@ class SloppyInterleaveDataset(dataset_ops.Dataset): cycle_length, dtype=dtypes.int64, name="cycle_length") self._block_length = ops.convert_to_tensor( block_length, dtype=dtypes.int64, name="block_length") + self._sloppy = ops.convert_to_tensor( + sloppy, dtype=dtypes.bool, name="sloppy") def _as_variant_tensor(self): - return gen_dataset_ops.sloppy_interleave_dataset( + return gen_dataset_ops.parallel_interleave_dataset( self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access self._map_func.captured_inputs, self._cycle_length, self._block_length, + self._sloppy, f=self._map_func, output_types=nest.flatten(self.output_types), output_shapes=nest.flatten(self.output_shapes)) @@ -82,6 +87,53 @@ class SloppyInterleaveDataset(dataset_ops.Dataset): return self._output_types +def parallel_interleave(map_func, cycle_length, block_length=1, sloppy=False): + """A parallel version of the `Dataset.interleave()` transformation. + + `parallel_interleave()` maps `map_func` across its input to produce nested + datasets, and outputs their elements interleaved. Unlike + @{tf.data.Dataset.interleave}, it gets elements from `cycle_length` nested + datasets in parallel, which increases the throughput, especially in the + presence of stragglers. Furthermore, the `sloppy` argument can be used to + improve performance, by relaxing the requirement that the outputs are produced + in a deterministic order, and allowing the implementation to skip over nested + datasets whose elements are not readily available when requested. + + Example usage: + + ```python + # Preprocess 4 files concurrently. + filenames = tf.data.Dataset.list_files("/path/to/data/train*.tfrecords") + dataset = filenames.apply( + tf.contrib.data.parallel_interleave( + lambda filename: tf.data.TFRecordDataset(filename), + cycle_length=4)) + ``` + + WARNING: If `sloppy` is `True`, the order of produced elements is not + deterministic. + + Args: + map_func: A function mapping a nested structure of tensors to a `Dataset`. + cycle_length: The number of threads to interleave from in parallel. + block_length: The number of consecutive elements to pull from a thread + before advancing to the next thread. + sloppy: If false, elements are produced in deterministic order. Otherwise, + the implementation is allowed, for the sake of expediency, to produce + elements in a non-deterministic order. + + Returns: + A `Dataset` transformation function, which can be passed to + @{tf.data.Dataset.apply}. + """ + def _apply_fn(dataset): + return ParallelInterleaveDataset( + dataset, map_func, cycle_length, block_length, sloppy) + return _apply_fn + + +@deprecation.deprecated( + None, "Use `tf.contrib.data.parallel_interleave(..., sloppy=True)`.") def sloppy_interleave(map_func, cycle_length, block_length=1): """A non-deterministic version of the `Dataset.interleave()` transformation. @@ -132,6 +184,6 @@ def sloppy_interleave(map_func, cycle_length, block_length=1): @{tf.data.Dataset.apply}. """ def _apply_fn(dataset): - return SloppyInterleaveDataset( - dataset, map_func, cycle_length, block_length) + return ParallelInterleaveDataset( + dataset, map_func, cycle_length, block_length, sloppy=True) return _apply_fn diff --git a/tensorflow/contrib/data/python/ops/iterator_ops.py b/tensorflow/contrib/data/python/ops/iterator_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..d736029fb035e573b70e8b19570e4e8ceca3c005 --- /dev/null +++ b/tensorflow/contrib/data/python/ops/iterator_ops.py @@ -0,0 +1,77 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Iterator ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import ops +from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.training import saver + + +def make_saveable_from_iterator(iterator): + """Returns a SaveableObject for saving/restore iterator state using Saver. + + Args: + iterator: Iterator. + + For example: + + ```python + with tf.Graph().as_default(): + ds = tf.data.Dataset.range(10) + iterator = ds.make_initializable_iterator() + # Build the iterator SaveableObject. + saveable_obj = tf.contrib.data.make_saveable_from_iterator(iterator) + # Add the SaveableObject to the SAVEABLE_OBJECTS collection so + # it can be automatically saved using Saver. + tf.add_to_collection(tf.GraphKeys.SAVEABLE_OBJECTS, saveable_obj) + saver = tf.train.Saver() + + while continue_training: + ... Perform training ... + if should_save_checkpoint: + saver.save() + ``` + + Note: When restoring the iterator, the existing iterator state is completely + discarded. This means that any changes you may have made to the Dataset + graph will be discarded as well! This includes the new Dataset graph + that you may have built during validation. So, while running validation, + make sure to run the initializer for the validation input pipeline after + restoring the checkpoint. + + Note: Not all iterators support checkpointing yet. Attempting to save the + state of an unsupported iterator will throw an error. + """ + return _Saveable(iterator._iterator_resource) # pylint: disable=protected-access + + +class _Saveable(saver.BaseSaverBuilder.SaveableObject): + """SaveableObject for saving/restoring iterator state.""" + + def __init__(self, iterator_resource): + serialized_iterator = gen_dataset_ops.serialize_iterator(iterator_resource) + specs = [ + saver.BaseSaverBuilder.SaveSpec(serialized_iterator, "", + iterator_resource.name + "-state") + ] + super(_Saveable, self).__init__(iterator_resource, specs, + iterator_resource.name) + + def restore(self, restored_tensors, unused_restored_shapes): + with ops.colocate_with(self.op): + return gen_dataset_ops.deserialize_iterator(self.op, restored_tensors[0]) diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..cfe8012b5657995b78d701528ea35cbb3748adb9 --- /dev/null +++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py @@ -0,0 +1,55 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Python wrapper for prefetching_ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.data.python.ops import gen_prefetching_ops +from tensorflow.contrib.util import loader +from tensorflow.python.platform import resource_loader + +_prefetching_ops = loader.load_op_library( + resource_loader.get_path_to_datafile("../../_prefetching_ops.so")) + + +# TODO(rohanj): Add a python class that constructs resource in the __init__ +# method and provides a get_next() that calls the prefetch op. +def function_buffering_resource(string_arg, + target_device, + shared_name, + f, + buffer_size, + thread_pool_size=1, + container="", + name=None): + return gen_prefetching_ops.function_buffering_resource( + string_arg=string_arg, + target_device=target_device, + shared_name=shared_name, + f=f, + buffer_size=buffer_size, + thread_pool_size=thread_pool_size, + container=container, + name=name) + + +def function_buffering_resource_get_next(function_buffer_resource, + output_types, + name=None): + return gen_prefetching_ops.function_buffering_resource_get_next( + function_buffer_resource=function_buffer_resource, + output_types=output_types, + name=name) diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD index 93770c37deeb53420b8473751329037a4bf99ed6..4a4f3789016bed5db475da81b2448b682f158353 100644 --- a/tensorflow/contrib/distributions/BUILD +++ b/tensorflow/contrib/distributions/BUILD @@ -18,14 +18,20 @@ py_library( "//tensorflow/contrib/linalg:linalg_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", + "//tensorflow/python:clip_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn_ops", + "//tensorflow/python:template", "//tensorflow/python:tensor_util", "//tensorflow/python:util", + "//tensorflow/python:variable_scope", "//tensorflow/python/ops/distributions", + "//tensorflow/python/ops/linalg", "//third_party/py/numpy", ], ) @@ -55,7 +61,9 @@ py_library( "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", "//tensorflow/python/ops/distributions", + "//tensorflow/python/ops/linalg", "//third_party/py/numpy", "@six_archive//:six", ], @@ -797,6 +805,25 @@ cuda_py_test( ], ) +cuda_py_test( + name = "gumbel_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/gumbel_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "@six_archive//:six", + "//tensorflow/contrib/linalg:linalg_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "inline_test", size = "small", @@ -835,6 +862,22 @@ cuda_py_test( ], ) +cuda_py_test( + name = "masked_autoregressive_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/masked_autoregressive_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "permute_test", size = "small", @@ -870,6 +913,22 @@ cuda_py_test( ], ) +cuda_py_test( + name = "reshape_test", + size = "small", + srcs = ["python/kernel_tests/bijectors/reshape_test.py"], + additional_deps = [ + ":bijectors_py", + ":distributions_py", + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:platform_test", + ], +) + cuda_py_test( name = "sigmoid_test", size = "small", diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py index f33cc1de0abc82a3a8974dba4459a55fb4c2e82c..16f6533e57347a5fe41b017c9855d216fba9da82 100644 --- a/tensorflow/contrib/distributions/__init__.py +++ b/tensorflow/contrib/distributions/__init__.py @@ -28,8 +28,11 @@ from tensorflow.contrib.distributions.python.ops.chi2 import * from tensorflow.contrib.distributions.python.ops.conditional_distribution import * from tensorflow.contrib.distributions.python.ops.conditional_transformed_distribution import * from tensorflow.contrib.distributions.python.ops.deterministic import * +from tensorflow.contrib.distributions.python.ops.distribution_util import fill_triangular from tensorflow.contrib.distributions.python.ops.distribution_util import matrix_diag_transform +from tensorflow.contrib.distributions.python.ops.distribution_util import reduce_weighted_logsumexp from tensorflow.contrib.distributions.python.ops.distribution_util import softplus_inverse +from tensorflow.contrib.distributions.python.ops.distribution_util import tridiag from tensorflow.contrib.distributions.python.ops.estimator import * from tensorflow.contrib.distributions.python.ops.geometric import * from tensorflow.contrib.distributions.python.ops.independent import * @@ -140,13 +143,14 @@ _allowed_symbols = [ 'RelaxedOneHotCategorical', 'kl_divergence', 'RegisterKL', - 'matrix_diag_transform', 'fill_triangular', + 'matrix_diag_transform', + 'reduce_weighted_logsumexp', + 'softplus_inverse', + 'tridiag', 'normal_conjugates_known_scale_posterior', 'normal_conjugates_known_scale_predictive', - 'softplus_inverse', 'percentile', - 'reduce_weighted_logsumexp', 'assign_moving_mean_variance', 'assign_log_moving_mean_exp', 'moving_mean_variance', diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py new file mode 100644 index 0000000000000000000000000000000000000000..9a905980c7581a86bbcda8c6c726da57c09fe4f8 --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/gumbel_test.py @@ -0,0 +1,70 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +from scipy import stats + +from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import Gumbel +from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite +from tensorflow.python.ops.distributions.bijector_test_util import assert_scalar_congruency +from tensorflow.python.platform import test + + +class GumbelBijectorTest(test.TestCase): + """Tests correctness of the Gumbel bijector.""" + + def testBijector(self): + with self.test_session(): + loc = 0.3 + scale = 5. + bijector = Gumbel(loc=loc, scale=scale, event_ndims=1, validate_args=True) + self.assertEqual("gumbel", bijector.name) + x = np.array([[[-3.], [0.], [0.5], [4.2], [12.]]], dtype=np.float32) + # Gumbel distribution + gumbel_dist = stats.gumbel_r(loc=loc, scale=scale) + y = gumbel_dist.cdf(x).astype(np.float32) + self.assertAllClose(y, bijector.forward(x).eval()) + self.assertAllClose(x, bijector.inverse(y).eval()) + self.assertAllClose( + # We should lose a dimension from calculating the determinant of the + # jacobian. + np.squeeze(gumbel_dist.logpdf(x), axis=2), + bijector.forward_log_det_jacobian(x).eval()) + self.assertAllClose( + -bijector.inverse_log_det_jacobian(y).eval(), + bijector.forward_log_det_jacobian(x).eval(), + rtol=1e-4, + atol=0.) + + def testScalarCongruency(self): + with self.test_session(): + assert_scalar_congruency( + Gumbel(loc=0.3, scale=20.), lower_x=1., upper_x=100., rtol=0.02) + + def testBijectiveAndFinite(self): + with self.test_session(): + bijector = Gumbel(loc=0., scale=3.0, event_ndims=0, validate_args=True) + x = np.linspace(-10., 10., num=10).astype(np.float32) + y = np.linspace(0.01, 0.99, num=10).astype(np.float32) + assert_bijective_and_finite(bijector, x, y, rtol=1e-3) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py new file mode 100644 index 0000000000000000000000000000000000000000..25a9b6f5fe2ed6d218d6b44650fce17fa89c0664 --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/masked_autoregressive_test.py @@ -0,0 +1,153 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for MaskedAutoregressiveFlow.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.distributions.python.ops import test_util +from tensorflow.contrib.distributions.python.ops.bijectors.invert import Invert +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import masked_autoregressive_default_template +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import MaskedAutoregressiveFlow +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive_impl import _gen_mask +from tensorflow.python.framework import constant_op +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import variables +from tensorflow.python.ops.distributions import normal as normal_lib +from tensorflow.python.ops.distributions import transformed_distribution as transformed_distribution_lib +from tensorflow.python.platform import test + + +class GenMaskTest(test.TestCase): + + def test346Exclusive(self): + expected_mask = np.array( + [[0, 0, 0, 0], + [0, 0, 0, 0], + [1, 0, 0, 0], + [1, 0, 0, 0], + [1, 1, 0, 0], + [1, 1, 0, 0]]) + mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="exclusive") + self.assertAllEqual(expected_mask, mask) + + def test346Inclusive(self): + expected_mask = np.array( + [[1, 0, 0, 0], + [1, 0, 0, 0], + [1, 1, 0, 0], + [1, 1, 0, 0], + [1, 1, 1, 0], + [1, 1, 1, 0]]) + mask = _gen_mask(num_blocks=3, n_in=4, n_out=6, mask_type="inclusive") + self.assertAllEqual(expected_mask, mask) + + +class MaskedAutoregressiveFlowTest(test_util.VectorDistributionTestHelpers, + test.TestCase): + + @property + def _autoregressive_flow_kwargs(self): + return { + "shift_and_log_scale_fn": masked_autoregressive_default_template( + hidden_layers=[2], shift_only=False), + "is_constant_jacobian": False, + } + + def testBijector(self): + x_ = np.arange(3 * 4 * 2).astype(np.float32).reshape(3, 4, 2) + with self.test_session() as sess: + ma = MaskedAutoregressiveFlow( + validate_args=True, + **self._autoregressive_flow_kwargs) + x = constant_op.constant(x_) + forward_x = ma.forward(x) + # Use identity to invalidate cache. + inverse_y = ma.inverse(array_ops.identity(forward_x)) + fldj = ma.forward_log_det_jacobian(x) + # Use identity to invalidate cache. + ildj = ma.inverse_log_det_jacobian(array_ops.identity(forward_x)) + variables.global_variables_initializer().run() + [ + forward_x_, + inverse_y_, + ildj_, + fldj_, + ] = sess.run([ + forward_x, + inverse_y, + ildj, + fldj, + ]) + self.assertEqual("masked_autoregressive_flow", ma.name) + self.assertAllClose(forward_x_, forward_x_, rtol=1e-6, atol=0.) + self.assertAllClose(x_, inverse_y_, rtol=1e-5, atol=0.) + self.assertAllClose(ildj_, -fldj_, rtol=1e-6, atol=0.) + + def testMutuallyConsistent(self): + dims = 4 + with self.test_session() as sess: + ma = MaskedAutoregressiveFlow( + validate_args=True, + **self._autoregressive_flow_kwargs) + dist = transformed_distribution_lib.TransformedDistribution( + distribution=normal_lib.Normal(loc=0., scale=1.), + bijector=ma, + event_shape=[dims], + validate_args=True) + self.run_test_sample_consistent_log_prob( + sess_run_fn=sess.run, + dist=dist, + num_samples=int(1e5), + radius=1., + center=0., + rtol=0.02) + + def testInvertMutuallyConsistent(self): + dims = 4 + with self.test_session() as sess: + ma = Invert(MaskedAutoregressiveFlow( + validate_args=True, + **self._autoregressive_flow_kwargs)) + dist = transformed_distribution_lib.TransformedDistribution( + distribution=normal_lib.Normal(loc=0., scale=1.), + bijector=ma, + event_shape=[dims], + validate_args=True) + self.run_test_sample_consistent_log_prob( + sess_run_fn=sess.run, + dist=dist, + num_samples=int(1e5), + radius=1., + center=0., + rtol=0.02) + + +class MaskedAutoregressiveFlowShiftOnlyTest(MaskedAutoregressiveFlowTest): + + @property + def _autoregressive_flow_kwargs(self): + return { + "shift_and_log_scale_fn": masked_autoregressive_default_template( + hidden_layers=[2], shift_only=True), + "is_constant_jacobian": True, + } + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py new file mode 100644 index 0000000000000000000000000000000000000000..38b3a23c2d684a6f89b7c4be4a763c649bf4de15 --- /dev/null +++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/reshape_test.py @@ -0,0 +1,242 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Reshape Bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.distributions.python.ops.bijectors.reshape import Reshape +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import array_ops +from tensorflow.python.ops.distributions.bijector_test_util import assert_bijective_and_finite +from tensorflow.python.platform import test + + +class ReshapeBijectorTest(test.TestCase): + """Tests correctness of the reshape transformation.""" + + def setUp(self): + self._rng = np.random.RandomState(42) + + def testBijector(self): + """Do a basic sanity check of forward, inverse, jacobian.""" + expected_x = np.random.randn(4, 3, 2) + expected_y = np.reshape(expected_x, [4, 6]) + + with self.test_session() as sess: + bijector = Reshape( + event_shape_out=[6,], + event_shape_in=[3, 2], + validate_args=True) + (x_, + y_, + fldj_, + ildj_) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + bijector.forward_log_det_jacobian(expected_x), + bijector.inverse_log_det_jacobian(expected_y), + )) + self.assertEqual("reshape", bijector.name) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + self.assertAllClose(0., fldj_, rtol=1e-6, atol=0) + self.assertAllClose(0., ildj_, rtol=1e-6, atol=0) + + def testEventShapeDynamicNdims(self): + """Check forward/inverse shape methods with dynamic ndims.""" + + shape_in = tensor_shape.TensorShape([6,]) + shape_in_ph = array_ops.placeholder(dtype=dtypes.int32) + + shape_out = tensor_shape.TensorShape([2, 3]) + shape_out_ph = array_ops.placeholder(dtype=dtypes.int32) + + bijector = Reshape( + event_shape_out=shape_out_ph, + event_shape_in=shape_in_ph, validate_args=True) + + # using the _tensor methods, we should always get a fully-specified + # result since these are evaluated at graph runtime. + with self.test_session() as sess: + (shape_out_, + shape_in_) = sess.run(( + bijector.forward_event_shape_tensor(shape_in), + bijector.inverse_event_shape_tensor(shape_out), + ), feed_dict={ + shape_in_ph: shape_in, + shape_out_ph: shape_out, + }) + self.assertAllEqual(shape_out, shape_out_) + self.assertAllEqual(shape_in, shape_in_) + + def testEventShapeDynamic(self): + """Check shape methods with static ndims but dynamic shape.""" + + shape_in = tensor_shape.TensorShape([6,]) + shape_in_partial = tensor_shape.TensorShape([None,]) + shape_in_ph = array_ops.placeholder( + shape=[1,], dtype=dtypes.int32) + + shape_out = tensor_shape.TensorShape([2, 3]) + shape_out_partial = tensor_shape.TensorShape([None, None]) + shape_out_ph = array_ops.placeholder( + shape=[2,], dtype=dtypes.int32) + + bijector = Reshape( + event_shape_out=shape_out_ph, + event_shape_in=shape_in_ph, + validate_args=True) + + # if event shapes are not statically available, should + # return partially-specified TensorShapes. + self.assertAllEqual( + bijector.forward_event_shape(shape_in).as_list(), + shape_out_partial.as_list()) + self.assertAllEqual( + bijector.inverse_event_shape(shape_out).as_list(), + shape_in_partial.as_list()) + + # using the _tensor methods, we should always get a fully-specified + # result since these are evaluated at graph runtime. + with self.test_session() as sess: + (shape_out_, + shape_in_) = sess.run(( + bijector.forward_event_shape_tensor(shape_in), + bijector.inverse_event_shape_tensor(shape_out), + ), feed_dict={ + shape_in_ph: shape_in, + shape_out_ph: shape_out, + }) + self.assertAllEqual(shape_out, shape_out_) + self.assertAllEqual(shape_in, shape_in_) + + def testEventShapeStatic(self): + """Check shape methods when shape is statically known.""" + + shape_in = tensor_shape.TensorShape([6,]) + shape_out = tensor_shape.TensorShape([2, 3]) + + bijector_static = Reshape( + event_shape_out=shape_out, + event_shape_in=shape_in, + validate_args=True) + + # test that forward_ and inverse_event_shape do sensible things + # when shapes are statically known. + self.assertEqual( + bijector_static.forward_event_shape(shape_in), + shape_out) + self.assertEqual( + bijector_static.inverse_event_shape(shape_out), + shape_in) + + with self.test_session() as sess: + (shape_out_static_, + shape_in_static_, + ) = sess.run(( + bijector_static.forward_event_shape_tensor(shape_in), + bijector_static.inverse_event_shape_tensor(shape_out), + )) + self.assertAllEqual(shape_out, shape_out_static_) + self.assertAllEqual(shape_in, shape_in_static_) + + def testScalarReshape(self): + """Test reshaping to and from a scalar shape ().""" + + expected_x = np.random.randn(4, 3, 1) + expected_y = np.reshape(expected_x, [4, 3]) + + expected_x_scalar = np.random.randn(1,) + expected_y_scalar = expected_x_scalar[0] + + with self.test_session() as sess: + bijector = Reshape( + event_shape_out=[], + event_shape_in=[1,], validate_args=True) + + (x_, + y_, + x_scalar_, + y_scalar_ + ) = sess.run(( + bijector.inverse(expected_y), + bijector.forward(expected_x), + bijector.inverse(expected_y_scalar), + bijector.forward(expected_x_scalar), + )) + self.assertAllClose(expected_y, y_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x, x_, rtol=1e-6, atol=0) + self.assertAllClose(expected_y_scalar, y_scalar_, rtol=1e-6, atol=0) + self.assertAllClose(expected_x_scalar, x_scalar_, rtol=1e-6, atol=0) + + def testRaisesOpError(self): + x1 = np.random.randn(4, 2, 3) + x2 = np.random.randn(4, 3, 2) + x3 = np.random.randn(4, 5, 1, 1) + + with self.test_session() as sess: + shape_in_ph = array_ops.placeholder(shape=[2,], dtype=dtypes.int32) + shape_out_ph = array_ops.placeholder(shape=[3,], dtype=dtypes.int32) + bijector = Reshape( + event_shape_out=shape_out_ph, + event_shape_in=shape_in_ph, + validate_args=True) + + with self.assertRaisesOpError( + "Input `event_shape` does not match `event_shape_in`."): + sess.run(bijector.forward(x2), + feed_dict={shape_out_ph: [1, 6, 1], + shape_in_ph: [2, 3]}) + + with self.assertRaisesOpError( + "event_shape_out entries must be positive."): + sess.run(bijector.forward(x1), + feed_dict={shape_out_ph: [-1, -1, 6], + shape_in_ph: [2, 3]}) + + # test that *all* methods check basic assertions + fd_mismatched = {shape_out_ph: [1, 1, 5], shape_in_ph: [2, 3]} + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.forward(x1), feed_dict=fd_mismatched) + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.inverse(x3), feed_dict=fd_mismatched) + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.inverse_log_det_jacobian(x3), + feed_dict=fd_mismatched) + with self.assertRaisesOpError( + "Input/output `event_size`s do not match."): + sess.run(bijector.forward_log_det_jacobian(x1), + feed_dict=fd_mismatched) + + def testBijectiveAndFinite(self): + x = np.random.randn(4, 2, 3) + y = np.reshape(x, [4, 1, 2, 3]) + with self.test_session(): + bijector = Reshape( + event_shape_in=[2, 3], + event_shape_out=[1, 2, 3], + validate_args=True) + assert_bijective_and_finite(bijector, x, y, rtol=1e-6, atol=0) + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py index 7a321db4b296e0f1f09874043a4568e6809f10fc..06318ca09dec851cf025fa35c83732b85824cbee 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/independent_test.py @@ -23,8 +23,10 @@ import numpy as np from tensorflow.contrib.distributions.python.ops import independent as independent_lib from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_diag_lib -from tensorflow.contrib.distributions.python.ops import test_util +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bernoulli as bernoulli_lib from tensorflow.python.ops.distributions import normal as normal_lib from tensorflow.python.platform import test from tensorflow.python.platform import tf_logging @@ -41,8 +43,10 @@ def try_import(name): # pylint: disable=invalid-name stats = try_import("scipy.stats") -class ProductDistributionTest( - test_util.VectorDistributionTestHelpers, test.TestCase): +class ProductDistributionTest(test.TestCase): + + def setUp(self): + self._rng = np.random.RandomState(42) def testSampleAndLogProbUnivariate(self): loc = np.float32([-1., 1]) @@ -50,9 +54,9 @@ class ProductDistributionTest( with self.test_session() as sess: ind = independent_lib.Independent( distribution=normal_lib.Normal(loc=loc, scale=scale), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) - x = ind.sample([4, 5]) + x = ind.sample([4, 5], seed=42) log_prob_x = ind.log_prob(x) x_, actual_log_prob_x = sess.run([x, log_prob_x]) @@ -73,9 +77,9 @@ class ProductDistributionTest( distribution=mvn_diag_lib.MultivariateNormalDiag( loc=loc, scale_identity_multiplier=scale), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) - x = ind.sample([4, 5]) + x = ind.sample([4, 5], seed=42) log_prob_x = ind.log_prob(x) x_, actual_log_prob_x = sess.run([x, log_prob_x]) @@ -98,7 +102,7 @@ class ProductDistributionTest( distribution=mvn_diag_lib.MultivariateNormalDiag( loc=loc, scale_identity_multiplier=scale), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) x = ind.sample(int(n_samp), seed=42) sample_mean = math_ops.reduce_mean(x, axis=0) @@ -122,6 +126,59 @@ class ProductDistributionTest( self.assertAllClose(sample_entropy_, actual_entropy_, rtol=0.01, atol=0.) self.assertAllClose(loc, actual_mode_, rtol=1e-6, atol=0.) + def _testMnistLike(self, static_shape): + sample_shape = [4, 5] + batch_shape = [10] + image_shape = [28, 28, 1] + logits = 3 * self._rng.random_sample( + batch_shape + image_shape).astype(np.float32) - 1 + + def expected_log_prob(x, logits): + return (x * logits - np.log1p(np.exp(logits))).sum(-1).sum(-1).sum(-1) + + with self.test_session() as sess: + logits_ph = array_ops.placeholder( + dtypes.float32, shape=logits.shape if static_shape else None) + ind = independent_lib.Independent( + distribution=bernoulli_lib.Bernoulli(logits=logits_ph)) + x = ind.sample(sample_shape, seed=42) + log_prob_x = ind.log_prob(x) + [ + x_, + actual_log_prob_x, + ind_batch_shape, + ind_event_shape, + x_shape, + log_prob_x_shape, + ] = sess.run([ + x, + log_prob_x, + ind.batch_shape_tensor(), + ind.event_shape_tensor(), + array_ops.shape(x), + array_ops.shape(log_prob_x), + ], feed_dict={logits_ph: logits}) + + if static_shape: + ind_batch_shape = ind.batch_shape + ind_event_shape = ind.event_shape + x_shape = x.shape + log_prob_x_shape = log_prob_x.shape + + self.assertAllEqual(batch_shape, ind_batch_shape) + self.assertAllEqual(image_shape, ind_event_shape) + self.assertAllEqual(sample_shape + batch_shape + image_shape, x_shape) + self.assertAllEqual(sample_shape + batch_shape, log_prob_x_shape) + self.assertAllClose(expected_log_prob(x_, logits), + actual_log_prob_x, + rtol=1e-6, atol=0.) + + def testMnistLikeStaticShape(self): + self._testMnistLike(static_shape=True) + + def testMnistLikeDynamicShape(self): + self._testMnistLike(static_shape=False) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py index ee4f989dac0761f04b1b6bc88f7de598f194634e..ece6bc077d9e21502fdfd01300a9d3e9f2c9c380 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/mixture_same_family_test.py @@ -94,10 +94,10 @@ class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers, loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, gm, radius=1., center=[-1., 1], rtol=0.02) + sess.run, gm, radius=1., center=[-1., 1], rtol=0.02) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, gm, radius=1., center=[1., -1], rtol=0.02) + sess.run, gm, radius=1., center=[1., -1], rtol=0.02) def testLogCdf(self): with self.test_session() as sess: @@ -122,7 +122,7 @@ class MixtureSameFamilyTest(test_util.VectorDistributionTestHelpers, mixture_distribution=categorical_lib.Categorical(probs=[0.3, 0.7]), components_distribution=mvn_diag_lib.MultivariateNormalDiag( loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5])) - self.run_test_sample_consistent_mean_covariance(sess, gm) + self.run_test_sample_consistent_mean_covariance(sess.run, gm) def testVarianceConsistentCovariance(self): with self.test_session() as sess: diff --git a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py index 43e302475b49ef5245ba324c35ca294b51a566b6..933756aa8e12cca4c42eb98d9193512bbf2ad585 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/mvn_diag_test.py @@ -289,6 +289,18 @@ class MultivariateNormalDiagTest(test.TestCase): self.assertListEqual(mvn.batch_shape.as_list(), [2, 3]) self.assertListEqual(mvn.event_shape.as_list(), [None]) + def testKLDivIdenticalGradientDefined(self): + dims = 3 + with self.test_session() as sess: + loc = array_ops.zeros([dims], dtype=dtypes.float32) + mvn = ds.MultivariateNormalDiag( + loc=loc, + scale_diag=np.ones([dims], dtype=np.float32)) + g = gradients_impl.gradients(ds.kl_divergence(mvn, mvn), loc) + g_ = sess.run(g) + self.assertAllEqual(np.ones_like(g_, dtype=np.bool), + np.isfinite(g_)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py index c1a74c6483b9843c609ac94054a8c27476f7d7ff..37edaa42cdc202cda4aa173752a3639792f96daf 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/negative_binomial_test.py @@ -241,6 +241,28 @@ class NegativeBinomialTest(test.TestCase): atol=0., rtol=.02) + def testLogProbOverflow(self): + with self.test_session() as sess: + logits = np.float32([20., 30., 40.]) + total_count = np.float32(1.) + x = np.float32(0.) + nb = negative_binomial.NegativeBinomial( + total_count=total_count, logits=logits) + log_prob_ = sess.run(nb.log_prob(x)) + self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool), + np.isfinite(log_prob_)) + + def testLogProbUnderflow(self): + with self.test_session() as sess: + logits = np.float32([-90, -100, -110]) + total_count = np.float32(1.) + x = np.float32(0.) + nb = negative_binomial.NegativeBinomial( + total_count=total_count, logits=logits) + log_prob_ = sess.run(nb.log_prob(x)) + self.assertAllEqual(np.ones_like(log_prob_, dtype=np.bool), + np.isfinite(log_prob_)) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py index 7cb46bb2367658518c98baaa14947b5ad837ff12..3c0147b8cf6e1b6a2791e85c0c0997992445fa7e 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/poisson_lognormal_test.py @@ -18,8 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import numpy as np + from tensorflow.contrib.distributions.python.ops import poisson_lognormal from tensorflow.contrib.distributions.python.ops import test_util +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -32,60 +36,80 @@ class PoissonLogNormalQuadratureCompoundTest( pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=-2., scale=1.1, - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( - sess, pln, rtol=0.1) + sess.run, pln, rtol=0.1) def testMeanVariance(self): with self.test_session() as sess: pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=0., scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( - sess, pln, rtol=0.02) + sess.run, pln, rtol=0.02) def testSampleProbConsistentBroadcastScalar(self): with self.test_session() as sess: pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( - sess, pln, rtol=0.1, atol=0.01) + sess.run, pln, rtol=0.1, atol=0.01) def testMeanVarianceBroadcastScalar(self): with self.test_session() as sess: pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( - sess, pln, rtol=0.1, atol=0.01) + sess.run, pln, rtol=0.1, atol=0.01) def testSampleProbConsistentBroadcastBoth(self): with self.test_session() as sess: pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[[0.], [-0.5]], scale=[[1., 0.9]], - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_log_prob( - sess, pln, rtol=0.1, atol=0.08) + sess.run, pln, rtol=0.1, atol=0.08) def testMeanVarianceBroadcastBoth(self): with self.test_session() as sess: pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( loc=[[0.], [-0.5]], scale=[[1., 0.9]], - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) self.run_test_sample_consistent_mean_variance( - sess, pln, rtol=0.1, atol=0.01) + sess.run, pln, rtol=0.1, atol=0.01) + + def testSampleProbConsistentDynamicQuadrature(self): + with self.test_session() as sess: + qgrid = array_ops.placeholder(dtype=dtypes.float32) + qprobs = array_ops.placeholder(dtype=dtypes.float32) + g, p = np.polynomial.hermite.hermgauss(deg=10) + pln = poisson_lognormal.PoissonLogNormalQuadratureCompound( + loc=-2., + scale=1.1, + quadrature_grid_and_probs=(g, p), + validate_args=True) + self.run_test_sample_consistent_log_prob( + lambda x: sess.run(x, feed_dict={qgrid: g, qprobs: p}), + pln, rtol=0.1) if __name__ == "__main__": diff --git a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py index 8c8363fe3f5159ed4def82472df8cb8ff518b05c..faae9da6ad812c629a2bdbb985fdd6f78a0860e1 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/relaxed_onehot_categorical_test.py @@ -164,6 +164,14 @@ class RelaxedOneHotCategoricalTest(test.TestCase): self.assertAllEqual([5, 3], dist.sample(5).eval(feed_dict=feed_dict).shape) + def testDTypes(self): + # check that sampling and log_prob work for a range of dtypes + with self.test_session(): + for dtype in (dtypes.float16, dtypes.float32, dtypes.float64): + logits = random_ops.random_uniform(shape=[3, 3], dtype=dtype) + dist = relaxed_onehot_categorical.RelaxedOneHotCategorical( + temperature=0.5, logits=logits) + dist.log_prob(dist.sample()) if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py index 4001530f6654a656891ebc15397cc3f618711bd8..103d8e186221e879d1734a097114708429f725bd 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py @@ -116,6 +116,18 @@ class TransformedDistributionTest(test.TestCase): np.log(sp_normal.pdf(2.13) + sp_normal.pdf(-2.13)), abs_normal.log_prob(2.13).eval()) + def testQuantile(self): + with self.test_session() as sess: + logit_normal = self._cls()( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.Sigmoid(), + validate_args=True) + grid = [0., 0.25, 0.5, 0.75, 1.] + q = logit_normal.quantile(grid) + cdf = logit_normal.cdf(q) + cdf_ = sess.run(cdf) + self.assertAllClose(grid, cdf_, rtol=1e-6, atol=0.) + def testCachedSamples(self): exp_forward_only = bs.Exp(event_ndims=0) exp_forward_only._inverse = self._make_unimplemented( diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py index aea4d4250383f5a6ae1af5545e06db08ac3788a3..de4a221f7badca8267a81d612a57137c676ff052 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_diffeomixture_test.py @@ -22,6 +22,8 @@ import numpy as np from tensorflow.contrib.distributions.python.ops import test_util from tensorflow.contrib.distributions.python.ops import vector_diffeomixture as vector_diffeomixture_lib +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.ops.distributions import normal as normal_lib from tensorflow.python.ops.linalg import linear_operator_diag as linop_diag_lib from tensorflow.python.ops.linalg import linear_operator_identity as linop_identity_lib @@ -55,10 +57,10 @@ class VectorDiffeomixtureTest( validate_args=True) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=2., center=0., rtol=0.005) + sess.run, vdm, radius=2., center=0., rtol=0.005) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=4., center=2., rtol=0.005) + sess.run, vdm, radius=4., center=2., rtol=0.005) def testSampleProbConsistentBroadcastMixNonStandardBase(self): with self.test_session() as sess: @@ -83,10 +85,10 @@ class VectorDiffeomixtureTest( validate_args=True) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=2., center=1., rtol=0.006) + sess.run, vdm, radius=2., center=1., rtol=0.006) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=4., center=3., rtol=0.009) + sess.run, vdm, radius=4., center=3., rtol=0.009) def testSampleProbConsistentBroadcastMixBatch(self): with self.test_session() as sess: @@ -114,10 +116,10 @@ class VectorDiffeomixtureTest( validate_args=True) # Ball centered at component0's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=2., center=0., rtol=0.005) + sess.run, vdm, radius=2., center=0., rtol=0.005) # Larger ball centered at component1's mean. self.run_test_sample_consistent_log_prob( - sess, vdm, radius=4., center=2., rtol=0.005) + sess.run, vdm, radius=4., center=2., rtol=0.005) def testMeanCovarianceNoBatch(self): with self.test_session() as sess: @@ -141,7 +143,7 @@ class VectorDiffeomixtureTest( ], validate_args=True) self.run_test_sample_consistent_mean_covariance( - sess, vdm, rtol=0.02, cov_rtol=0.06) + sess.run, vdm, rtol=0.02, cov_rtol=0.06) def testMeanCovarianceNoBatchUncenteredNonStandardBase(self): with self.test_session() as sess: @@ -165,7 +167,7 @@ class VectorDiffeomixtureTest( ], validate_args=True) self.run_test_sample_consistent_mean_covariance( - sess, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025) + sess.run, vdm, num_samples=int(1e6), rtol=0.01, cov_atol=0.025) def testMeanCovarianceBatch(self): with self.test_session() as sess: @@ -192,7 +194,40 @@ class VectorDiffeomixtureTest( ], validate_args=True) self.run_test_sample_consistent_mean_covariance( - sess, vdm, rtol=0.02, cov_rtol=0.06) + sess.run, vdm, rtol=0.02, cov_rtol=0.06) + + def testSampleProbConsistentDynamicQuadrature(self): + with self.test_session() as sess: + qgrid = array_ops.placeholder(dtype=dtypes.float32) + qprobs = array_ops.placeholder(dtype=dtypes.float32) + g, p = np.polynomial.hermite.hermgauss(deg=8) + dims = 4 + vdm = vector_diffeomixture_lib.VectorDiffeomixture( + mix_loc=[[0.], [1.]], + mix_scale=[1.], + distribution=normal_lib.Normal(0., 1.), + loc=[ + None, + np.float32([2.]*dims), + ], + scale=[ + linop_identity_lib.LinearOperatorScaledIdentity( + num_rows=dims, + multiplier=np.float32(1.1), + is_positive_definite=True), + linop_diag_lib.LinearOperatorDiag( + diag=np.linspace(2.5, 3.5, dims, dtype=np.float32), + is_positive_definite=True), + ], + quadrature_grid_and_probs=(g, p), + validate_args=True) + # Ball centered at component0's mean. + sess_run_fn = lambda x: sess.run(x, feed_dict={qgrid: g, qprobs: p}) + self.run_test_sample_consistent_log_prob( + sess_run_fn, vdm, radius=2., center=0., rtol=0.005) + # Larger ball centered at component1's mean. + self.run_test_sample_consistent_log_prob( + sess_run_fn, vdm, radius=4., center=2., rtol=0.005) # TODO(jvdillon): We've tested that (i) .sample and .log_prob are consistent, # (ii) .mean, .stddev etc... and .sample are consistent. However, we haven't diff --git a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py index a5d837d4541b63922aea2fcdf648898b391c662d..2bc6a926dd66fd2b5796576c723345ca2014aad6 100644 --- a/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py +++ b/tensorflow/contrib/distributions/python/kernel_tests/vector_sinh_arcsinh_diag_test.py @@ -210,15 +210,15 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers, validate_args=True) self.run_test_sample_consistent_log_prob( - sess, sasnorm, radius=1.0, center=0., rtol=0.1) + sess.run, sasnorm, radius=1.0, center=0., rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=-0.15, rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=0.15, @@ -237,15 +237,15 @@ class VectorSinhArcsinhDiagTest(test_util.VectorDistributionTestHelpers, validate_args=True) self.run_test_sample_consistent_log_prob( - sess, sasnorm, radius=1.0, center=0., rtol=0.1) + sess.run, sasnorm, radius=1.0, center=0., rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=-0.15, rtol=0.1) self.run_test_sample_consistent_log_prob( - sess, + sess.run, sasnorm, radius=1.0, center=0.15, diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py index c9ed546a3476d01703188b1da147b09424caf592..bc0ec7f195af009c87020ce8c4ea18f2e713759a 100644 --- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py +++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py @@ -22,17 +22,23 @@ @@CholeskyOuterProduct @@ConditionalBijector @@Exp +@@Gumbel @@Identity @@Inline @@Invert +@@MaskedAutoregressiveFlow @@Permute @@PowerTransform +@@Reshape @@Sigmoid @@SigmoidCentered @@SinhArcsinh @@SoftmaxCentered @@Softplus @@Weibull + +@@masked_autoregressive_default_template +@@masked_dense """ from __future__ import absolute_import @@ -48,10 +54,13 @@ from tensorflow.contrib.distributions.python.ops.bijectors.chain import * from tensorflow.contrib.distributions.python.ops.bijectors.cholesky_outer_product import * from tensorflow.contrib.distributions.python.ops.bijectors.conditional_bijector import * from tensorflow.contrib.distributions.python.ops.bijectors.exp import * +from tensorflow.contrib.distributions.python.ops.bijectors.gumbel import * from tensorflow.contrib.distributions.python.ops.bijectors.inline import * from tensorflow.contrib.distributions.python.ops.bijectors.invert import * +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive import * from tensorflow.contrib.distributions.python.ops.bijectors.permute import * from tensorflow.contrib.distributions.python.ops.bijectors.power_transform import * +from tensorflow.contrib.distributions.python.ops.bijectors.reshape import * from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import * from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid_centered import * from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import * diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py new file mode 100644 index 0000000000000000000000000000000000000000..cf37aa51115ed98ab263bc03bcb297a03432a7ae --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel.py @@ -0,0 +1,29 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gumbel bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.contrib.distributions.python.ops.bijectors.gumbel_impl import * +# pylint: enable=wildcard-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = ["Gumbel"] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..67f39785563255be0fe154aca3cbcf01c6a01e73 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/gumbel_impl.py @@ -0,0 +1,124 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Gumbel bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bijector + +__all__ = [ + "Gumbel", +] + + +class Gumbel(bijector.Bijector): + """Compute `Y = g(X) = exp(-exp(-(X - loc) / scale))`. + + This bijector maps inputs from `[-inf, inf]` to [0, 1]`. The inverse of the + bijector applied to a uniform random variable `X ~ U(0, 1) gives back a + random variable with the + [Gumbel distribution](https://en.wikipedia.org/wiki/Gumbel_distribution): + + ```none + Y ~ Gumbel(loc, scale) + pdf(y; loc, scale) = exp( + -( (y - loc) / scale + exp(- (y - loc) / scale) ) ) / scale + ``` + """ + + def __init__(self, + loc=0., + scale=1., + event_ndims=0, + validate_args=False, + name="gumbel"): + """Instantiates the `Gumbel` bijector. + + Args: + loc: Float-like `Tensor` that is the same dtype and is + broadcastable with `scale`. + This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. + scale: Positive Float-like `Tensor` that is the same dtype and is + broadcastable with `loc`. + This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`. + event_ndims: Python scalar indicating the number of dimensions associated + with a particular draw from the distribution. + validate_args: Python `bool` indicating whether arguments should be + checked for correctness. + name: Python `str` name given to ops managed by this object. + """ + self._graph_parents = [] + self._name = name + self._validate_args = validate_args + with self._name_scope("init", values=[loc, scale]): + self._loc = ops.convert_to_tensor(loc, name="loc") + self._scale = ops.convert_to_tensor(scale, name="scale") + check_ops.assert_same_float_dtype([self._loc, self._scale]) + if validate_args: + self._scale = control_flow_ops.with_dependencies([ + check_ops.assert_positive( + self._scale, message="Argument scale was not positive") + ], self._scale) + + super(Gumbel, self).__init__( + event_ndims=event_ndims, validate_args=validate_args, name=name) + + @property + def loc(self): + """The `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.""" + return self._loc + + @property + def scale(self): + """This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.""" + return self._scale + + def _forward(self, x): + z = (x - self.loc) / self.scale + return math_ops.exp(-math_ops.exp(-z)) + + def _inverse(self, y): + y = self._maybe_assert_valid_y(y) + return self.loc - self.scale * math_ops.log(-math_ops.log(y)) + + def _inverse_log_det_jacobian(self, y): + y = self._maybe_assert_valid_y(y) + event_dims = self._event_dims_tensor(y) + return math_ops.reduce_sum( + math_ops.log(self.scale / (-math_ops.log(y) * y)), axis=event_dims) + + def _forward_log_det_jacobian(self, x): + event_dims = self._event_dims_tensor(x) + z = (x - self.loc) / self.scale + return math_ops.reduce_sum( + -z - math_ops.exp(-z) - math_ops.log(self.scale), axis=event_dims) + + def _maybe_assert_valid_y(self, y): + if not self.validate_args: + return y + is_positive = check_ops.assert_non_negative( + y, message="Inverse transformation input must be greater than 0.") + less_than_one = check_ops.assert_less_equal( + y, + constant_op.constant(1., y.dtype), + message="Inverse transformation input must be less than or equal to 1.") + return control_flow_ops.with_dependencies([is_positive, less_than_one], y) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py new file mode 100644 index 0000000000000000000000000000000000000000..132dc570f94719b6c71fb269866c943774481b7e --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py @@ -0,0 +1,33 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MaskedAutoregressiveFlow bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.contrib.distributions.python.ops.bijectors.masked_autoregressive_impl import * +# pylint: enable=wildcard-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = [ + "MaskedAutoregressiveFlow", + "masked_dense", + "masked_autoregressive_default_template", +] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..ae142883931274b594dbbafbe86bd71e75c621bc --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive_impl.py @@ -0,0 +1,473 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""MaskedAutoregressiveFlow bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.layers import core as layers +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import clip_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import template as template_ops +from tensorflow.python.ops import variable_scope as variable_scope_lib +from tensorflow.python.ops.distributions import bijector as bijector_lib + + +__all__ = [ + "MaskedAutoregressiveFlow", + "masked_autoregressive_default_template", + "masked_dense", +] + + +class MaskedAutoregressiveFlow(bijector_lib.Bijector): + """Affine MaskedAutoregressiveFlow bijector for vector-valued events. + + The affine autoregressive flow [1] provides a relatively simple framework for + user-specified (deep) architectures to learn a distribution over vector-valued + events. Regarding terminology, + + "Autoregressive models decompose the joint density as a product of + conditionals, and model each conditional in turn. Normalizing flows + transform a base density (e.g. a standard Gaussian) into the target density + by an invertible transformation with tractable Jacobian." [1] + + In other words, the "autoregressive property" is equivalent to the + decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided + `shift_and_log_scale_fn`, `masked_autoregressive_default_template`, achieves + this property by zeroing out weights in its `masked_dense` layers. + + In the `tf.distributions` framework, a "normalizing flow" is implemented as a + `tf.distributions.bijectors.Bijector`. The `forward` "autoregression" + is implemented using a `tf.while_loop` and a deep neural network (DNN) with + masked weights such that the autoregressive property is automatically met in + the `inverse`. + + A `TransformedDistribution` using `MaskedAutoregressiveFlow(...)` uses the + (expensive) forward-mode calculation to draw samples and the (cheap) + reverse-mode calculation to compute log-probabilities. Conversely, a + `TransformedDistribution` using `Invert(MaskedAutoregressiveFlow(...))` uses + the (expensive) forward-mode calculation to compute log-probabilities and the + (cheap) reverse-mode calculation to compute samples. See "Example Use" + [below] for more details. + + Given a `shift_and_log_scale_fn`, the forward and inverse transformations are + (a sequence of) affine transformations. A "valid" `shift_and_log_scale_fn` + must compute each `shift` (aka `loc` or "mu" [2]) and `log(scale)` (aka + "alpha" [2]) such that each are broadcastable with the arguments to `forward` + and `inverse`, i.e., such that the calculations in `forward`, `inverse` + [below] are possible. + + For convenience, `masked_autoregressive_default_template` is offered as a + possible `shift_and_log_scale_fn` function. It implements the MADE + architecture [2]. MADE is a feed-forward network that computes a `shift` and + `log(scale)` using `masked_dense` layers in a deep neural network. Weights are + masked to ensure the autoregressive property. It is possible that this + architecture is suboptimal for your task. To build alternative networks, + either change the arguments to `masked_autoregressive_default_template`, use + the `masked_dense` function to roll-out your own, or use some other + architecture, e.g., using `tf.layers`. + + Warning: no attempt is made to validate that the `shift_and_log_scale_fn` + enforces the "autoregressive property". + + Assuming `shift_and_log_scale_fn` has valid shape and autoregressive + semantics, the forward transformation is, + + ```python + def forward(x): + y = zeros_like(x) + event_size = x.shape[-1] + for _ in range(event_size): + shift, log_scale = shift_and_log_scale_fn(y) + y = x * math_ops.exp(log_scale) + shift + return y + ``` + + and the inverse transformation is, + + ```python + def inverse(y): + shift, log_scale = shift_and_log_scale_fn(y) + return (y - shift) / math_ops.exp(log_scale) + ``` + + Notice that the `inverse` does not need a for-loop. This is because in the + forward pass each calculation of `shift` and `log_scale` is based on the `y` + calculated so far (not `x`). In the `inverse`, the `y` is fully known, thus is + equivalent to the scaling used in `forward` after `event_size` passes, i.e., + the "last" `y` used to compute `shift`, `log_scale`. (Roughly speaking, this + also proves the transform is bijective.) + + #### Example Use + + ```python + ds = tf.contrib.distributions + bs = tf.contrib.distributions.bijectors + + dims = 5 + + # A common choice for a normalizing flow is to use a Gaussian for the base + # distribution. (However, any continuous distribution would work.) E.g., + maf = ds.TransformedDistribution( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.MaskedAutoregressiveFlow( + shift_and_log_scale_fn=bs.masked_autoregressive_default_template( + hidden_layers=[512, 512])), + event_shape=[dims]) + + x = maf.sample() # Expensive; uses `tf.while_loop`, no Bijector caching. + maf.log_prob(x) # Almost free; uses Bijector caching. + maf.log_prob(0.) # Cheap; no `tf.while_loop` despite no Bijector caching. + + # [1] also describes an "Inverse Autoregressive Flow", e.g., + iaf = ds.TransformedDistribution( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.Invert(bs.MaskedAutoregressiveFlow( + shift_and_log_scale_fn=bs.masked_autoregressive_default_template( + hidden_layers=[512, 512]))), + event_shape=[dims]) + + x = iaf.sample() # Cheap; no `tf.while_loop` despite no Bijector caching. + iaf.log_prob(x) # Almost free; uses Bijector caching. + iaf.log_prob(0.) # Expensive; uses `tf.while_loop`, no Bijector caching. + + # In many (if not most) cases the default `shift_and_log_scale_fn` will be a + # poor choice. Here's an example of using a "shift only" version and with a + # different number/depth of hidden layers. + shift_only = True + maf_no_scale_hidden2 = ds.TransformedDistribution( + distribution=ds.Normal(loc=0., scale=1.), + bijector=bs.MaskedAutoregressiveFlow( + bs.masked_autoregressive_default_template( + hidden_layers=[32], + shift_only=shift_only), + is_constant_jacobian=shift_only), + event_shape=[dims]) + ``` + + [1]: "Masked Autoregressive Flow for Density Estimation." + George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017. + https://arxiv.org/abs/1705.07057 + + [2]: "MADE: Masked Autoencoder for Distribution Estimation." + Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015. + https://arxiv.org/abs/1502.03509 + + """ + + def __init__(self, + shift_and_log_scale_fn, + is_constant_jacobian=False, + validate_args=False, + name=None): + """Creates the MaskedAutoregressiveFlow bijector. + + Args: + shift_and_log_scale_fn: Python `callable` which computes `shift` and + `log_scale` from both the forward domain (`x`) and the inverse domain + (`y`). Calculation must respect the "autoregressive property" (see class + docstring). Suggested default + `masked_autoregressive_default_template(hidden_layers=...)`. + Typically the function contains `tf.Variables` and is wrapped using + `tf.make_template`. Returning `None` for either (both) `shift`, + `log_scale` is equivalent to (but more efficient than) returning zero. + is_constant_jacobian: Python `bool`. Default: `False`. When `True` the + implementation assumes `log_scale` does not depend on the forward domain + (`x`) or inverse domain (`y`) values. (No validation is made; + `is_constant_jacobian=False` is always safe but possibly computationally + inefficient.) + validate_args: Python `bool` indicating whether arguments should be + checked for correctness. + name: Python `str`, name given to ops managed by this object. + """ + name = name or "masked_autoregressive_flow" + self._shift_and_log_scale_fn = shift_and_log_scale_fn + super(MaskedAutoregressiveFlow, self).__init__( + is_constant_jacobian=is_constant_jacobian, + validate_args=validate_args, + name=name) + + def _forward(self, x): + event_size = array_ops.shape(x)[-1] + def _loop_body(index, y0): + """While-loop body for autoregression calculation.""" + # Set caching device to avoid re-getting the tf.Variable for every while + # loop iteration. + with variable_scope_lib.variable_scope( + variable_scope_lib.get_variable_scope()) as vs: + if vs.caching_device is None: + vs.set_caching_device(lambda op: op.device) + shift, log_scale = self._shift_and_log_scale_fn(y0) + y = x + if log_scale is not None: + y *= math_ops.exp(log_scale) + if shift is not None: + y += shift + return index + 1, y + _, y = control_flow_ops.while_loop( + cond=lambda index, _: index < event_size, + body=_loop_body, + loop_vars=[0, array_ops.zeros_like(x, name="y0")]) + return y + + def _inverse(self, y): + shift, log_scale = self._shift_and_log_scale_fn(y) + x = y + if shift is not None: + x -= shift + if log_scale is not None: + x *= math_ops.exp(-log_scale) + return x + + def _inverse_log_det_jacobian(self, y): + _, log_scale = self._shift_and_log_scale_fn(y) + if log_scale is None: + return constant_op.constant(0., dtype=y.dtype, name="ildj") + return -math_ops.reduce_sum(log_scale, axis=-1) + + +MASK_INCLUSIVE = "inclusive" +MASK_EXCLUSIVE = "exclusive" + + +def _gen_slices(num_blocks, n_in, n_out, mask_type=MASK_EXCLUSIVE): + """Generate the slices for building an autoregressive mask.""" + # TODO(b/67594795): Better support of dynamic shape. + slices = [] + col = 0 + d_in = n_in // num_blocks + d_out = n_out // num_blocks + row = d_out if mask_type == MASK_EXCLUSIVE else 0 + for _ in range(num_blocks): + row_slice = slice(row, None) + col_slice = slice(col, col + d_in) + slices.append([row_slice, col_slice]) + col += d_in + row += d_out + return slices + + +def _gen_mask(num_blocks, + n_in, + n_out, + mask_type=MASK_EXCLUSIVE, + dtype=dtypes.float32): + """Generate the mask for building an autoregressive dense layer.""" + # TODO(b/67594795): Better support of dynamic shape. + mask = np.zeros([n_out, n_in], dtype=dtype.as_numpy_dtype()) + slices = _gen_slices(num_blocks, n_in, n_out, mask_type=mask_type) + for [row_slice, col_slice] in slices: + mask[row_slice, col_slice] = 1 + return mask + + +def masked_dense(inputs, + units, + num_blocks=None, + exclusive=False, + kernel_initializer=None, + reuse=None, + name=None, + *args, + **kwargs): + """A autoregressively masked dense layer. Analogous to `tf.layers.dense`. + + See [1] for detailed explanation. + + [1]: "MADE: Masked Autoencoder for Distribution Estimation." + Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015. + https://arxiv.org/abs/1502.03509 + + Arguments: + inputs: Tensor input. + units: Python `int` scalar representing the dimensionality of the output + space. + num_blocks: Python `int` scalar representing the number of blocks for the + MADE masks. + exclusive: Python `bool` scalar representing whether to zero the diagonal of + the mask, used for the first layer of a MADE. + kernel_initializer: Initializer function for the weight matrix. + If `None` (default), weights are initialized using the + `tf.glorot_random_initializer`. + reuse: Python `bool` scalar representing whether to reuse the weights of a + previous layer by the same name. + name: Python `str` used to describe ops managed by this function. + *args: `tf.layers.dense` arguments. + **kwargs: `tf.layers.dense` keyword arguments. + + Returns: + Output tensor. + + Raises: + NotImplementedError: if rightmost dimension of `inputs` is unknown prior to + graph execution. + """ + # TODO(b/67594795): Better support of dynamic shape. + input_depth = inputs.shape.with_rank_at_least(1)[-1].value + if input_depth is None: + raise NotImplementedError( + "Rightmost dimension must be known prior to graph execution.") + + mask = _gen_mask(num_blocks, input_depth, units, + MASK_EXCLUSIVE if exclusive else MASK_INCLUSIVE).T + + if kernel_initializer is None: + kernel_initializer = init_ops.glorot_normal_initializer() + + def masked_initializer(shape, dtype=None, partition_info=None): + return mask * kernel_initializer(shape, dtype, partition_info) + + with ops.name_scope(name, "masked_dense", [inputs, units, num_blocks]): + layer = layers.Dense( + units, + kernel_initializer=masked_initializer, + kernel_constraint=lambda x: mask * x, + name=name, + dtype=inputs.dtype.base_dtype, + _scope=name, + _reuse=reuse, + *args, + **kwargs) + return layer.apply(inputs) + + +def masked_autoregressive_default_template( + hidden_layers, + shift_only=False, + activation=nn_ops.relu, + log_scale_min_clip=-5., + log_scale_max_clip=3., + log_scale_clip_gradient=False, + name=None, + *args, + **kwargs): + """Build the MADE Model [1]. + + This will be wrapped in a make_template to ensure the variables are only + created once. It takes the input and returns the `loc` ("mu" [1]) and + `log_scale` ("alpha" [1]) from the MADE network. + + Warning: This function uses `masked_dense` to create randomly initialized + `tf.Variables`. It is presumed that these will be fit, just as you would any + other neural architecture which uses `tf.layers.dense`. + + #### About Hidden Layers: + + Each element of `hidden_layers` should be greater than the `input_depth` + (i.e., `input_depth = tf.shape(input)[-1]` where `input` is the input to the + neural network). This is necessary to ensure the autoregressivity property. + + #### About Clipping: + + This function also optionally clips the `log_scale` (but possibly not its + gradient). This is useful because if `log_scale` is too small/large it might + underflow/overflow making it impossible for the `MaskedAutoregressiveFlow` + bijector to implement a bijection. Additionally, the `log_scale_clip_gradient` + `bool` indicates whether the gradient should also be clipped. The default does + not clip the gradient; this is useful because it still provides gradient + information (for fitting) yet solves the numerical stability problem. I.e., + `log_scale_clip_gradient = False` means + `grad[exp(clip(x))] = grad[x] exp(clip(x))` rather than the usual + `grad[clip(x)] exp(clip(x))`. + + [1]: "MADE: Masked Autoencoder for Distribution Estimation." + Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015. + https://arxiv.org/abs/1502.03509 + + Arguments: + hidden_layers: Python `list`-like of non-negative integer, scalars + indicating the number of units in each hidden layer. Default: `[512, 512]. + shift_only: Python `bool` indicating if only the `shift` term shall be + computed. Default: `False`. + activation: Activation function (callable). Explicitly setting to `None` + implies a linear activation. + log_scale_min_clip: `float`-like scalar `Tensor`, or a `Tensor` with the + same shape as `log_scale`. The minimum value to clip by. Default: -5. + log_scale_max_clip: `float`-like scalar `Tensor`, or a `Tensor` with the + same shape as `log_scale`. The maximum value to clip by. Default: 3. + log_scale_clip_gradient: Python `bool` indicating that the gradient of + `tf.clip_by_value` should be preserved. Default: `False`. + name: A name for ops managed by this function. Default: + "masked_autoregressive_default_template". + *args: `tf.layers.dense` arguments. + **kwargs: `tf.layers.dense` keyword arguments. + + Returns: + shift: `Float`-like `Tensor` of shift terms (the "mu" in [2]). + log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in [2]). + + Raises: + NotImplementedError: if rightmost dimension of `inputs` is unknown prior to + graph execution. + """ + + with ops.name_scope(name, "masked_autoregressive_default_template", + values=[log_scale_min_clip, log_scale_max_clip]): + def _fn(x): + """MADE parameterized via `masked_autoregressive_default_template`.""" + # TODO(b/67594795): Better support of dynamic shape. + input_depth = x.shape.with_rank_at_least(1)[-1].value + if input_depth is None: + raise NotImplementedError( + "Rightmost dimension must be known prior to graph execution.") + input_shape = (np.int32(x.shape.as_list()) if x.shape.is_fully_defined() + else array_ops.shape(x)) + for i, units in enumerate(hidden_layers): + x = masked_dense( + inputs=x, + units=units, + num_blocks=input_depth, + exclusive=True if i == 0 else False, + activation=activation, + *args, + **kwargs) + x = masked_dense( + inputs=x, + units=(1 if shift_only else 2) * input_depth, + num_blocks=input_depth, + activation=None, + *args, + **kwargs) + if shift_only: + x = array_ops.reshape(x, shape=input_shape) + return x, None + x = array_ops.reshape( + x, shape=array_ops.concat([input_shape, [2]], axis=0)) + shift, log_scale = array_ops.unstack(x, num=2, axis=-1) + which_clip = (math_ops.clip_by_value if log_scale_clip_gradient + else _clip_by_value_preserve_grad) + log_scale = which_clip(log_scale, log_scale_min_clip, log_scale_max_clip) + return shift, log_scale + return template_ops.make_template( + "masked_autoregressive_default_template", _fn) + + +def _clip_by_value_preserve_grad(x, clip_value_min, clip_value_max, name=None): + """Clips input while leaving gradient unaltered.""" + with ops.name_scope(name, "clip_by_value_preserve_grad", + [x, clip_value_min, clip_value_max]): + clip_x = clip_ops.clip_by_value(x, clip_value_min, clip_value_max) + return x + array_ops.stop_gradient(clip_x - x) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py new file mode 100644 index 0000000000000000000000000000000000000000..8997f7ab6929745275edb38712a5bbb0a9b25ddb --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape.py @@ -0,0 +1,29 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reshape bijector.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.contrib.distributions.python.ops.bijectors.reshape_impl import * +# pylint: enable=wildcard-import +from tensorflow.python.util.all_util import remove_undocumented + +_allowed_symbols = ["Reshape"] + +remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..93682639aa3be3b8f59a369dedb6ee773c468130 --- /dev/null +++ b/tensorflow/contrib/distributions/python/ops/bijectors/reshape_impl.py @@ -0,0 +1,297 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Reshape bijectors.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import check_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops.distributions import bijector as bijector_lib + + +__all__ = [ + "Reshape", +] + + +class Reshape(bijector_lib.Bijector): + """Reshapes the `event_shape` of a `Tensor`. + + The semantics generally follow that of `tf.reshape()`, with + a few differences: + * The user must provide both the input and output shape, so that + the transformation can be inverted. + * The `Reshape` bijector automatically broadcasts over the leftmost + dimensions of its input (`sample_shape` and `batch_shape`); only + the rightmost `event_ndims_in` dimensions are reshaped. The + number of dimensions to reshape is inferred from the provided + `event_shape_in` (`event_ndims_in = len(event_shape_in)`). + * The `Reshape` bijector does not currently support + partially-specified shapes, i.e., those with a dimension + implicitly specified by `-1`. + + Example usage: + ```python + + bs = tf.contrib.distributions.bijectors + + reverse = bs.Reshape(event_shape_out=[1,2], + event_shape_in=[2,]) + + reverse.forward([1., 2.]) # shape [2,] + # ==> [[1., 2.]] # shape [1,2] + + reverse.forward([[1., 2.], [3., 4.]]) # shape [2, 2] + # ==> [[[1., 2.]], [[3., 4.]]] # shape [2, 1, 2] + + reverse.inverse([[1., 2.]]) # shape [1,2] + # ==> [1., 2.] # shape [2,] + + reverse.forward_log_det_jacobian(any_value) + # ==> 0. + + reverse.inverse_log_det_jacobian(any_value) + # ==> 0. + ``` + + """ + + def __init__(self, event_shape_out, event_shape_in, + validate_args=False, name=None): + """Creates a `Reshape` bijector. + + Args: + event_shape_out: An `int`-like vector-shaped `Tensor` + representing the fully specified (no -1's) event shape of the + transformed output. + event_shape_in: An `int`-like vector-shaped `Tensor` + representing the fully specified (no -1's) event shape of the + input. + validate_args: Python `bool` indicating whether arguments should + be checked for correctness. + name: Python `str`, name given to ops managed by this object. + + Raises: + TypeError: if either `event_shape_in` or `event_shape_out` has + non-vector shape (`rank > 1`), or non-integer `dtype`. + ValueError: if either `event_shape_in` or `event_shape_out` + contains non-positive entries, or if their sizes do not match + (`prod(event_shape_in)` != `prod(event_shape_out)`), or if + their dimensionality(s) cannot be statically inferred. + """ + with ops.name_scope(name, "reshape", + values=[event_shape_out, event_shape_in]): + + event_shape_out = ops.convert_to_tensor(event_shape_out, + name="event_shape_out", + preferred_dtype=dtypes.int32) + event_shape_in = ops.convert_to_tensor(event_shape_in, + name="event_shape_in", + preferred_dtype=dtypes.int32) + + # check that input shapes are positive integers + assertions = [] + assertions += self._maybe_check_valid_shape( + event_shape_out, "event_shape_out", + validate_args=validate_args) + assertions += self._maybe_check_valid_shape( + event_shape_in, "event_shape_in", validate_args=validate_args) + + # check that prod(event_shape_in) = prod(event_shape_out) + assertions += self._maybe_check_matching_sizes( + event_shape_in, event_shape_out, validate_args=validate_args) + + self._assertions = assertions + self._event_shape_in = event_shape_in + self._event_shape_out = event_shape_out + self._event_shape_in_static = tensor_util.constant_value_as_shape( + event_shape_in) + self._event_shape_out_static = tensor_util.constant_value_as_shape( + event_shape_out) + + super(Reshape, self).__init__(is_constant_jacobian=True, + validate_args=validate_args, + name=name or "reshape") + + def _maybe_check_valid_shape(self, shape_tensor, label, + validate_args=False): + """Check that a shape Tensor is int-type and positive.""" + + assertions = [] + + if not shape_tensor.dtype.is_integer: + raise TypeError("{} dtype ({}) should be `int`-like.".format( + label, shape_tensor.dtype.name)) + + shape_rank = tensor_util.constant_value(array_ops.rank(shape_tensor)) + if shape_rank is not None and shape_rank > 1: + raise ValueError("{} rank should be <= 1.".format(label)) + + s = tensor_util.constant_value(shape_tensor) + if s is not None: + if (s <= 0).any(): + raise ValueError("{} entries must be positive, but found {}".format( + label, s)) + elif validate_args: + assertions.append(check_ops.assert_positive( + shape_tensor, message="{} entries must be positive".format(label))) + + return assertions + + def _maybe_check_matching_sizes(self, event_shape_in, event_shape_out, + validate_args=False): + """Check that prod(event_shape_in)==prod(event_shape_out).""" + + def _get_size_from_shape(shape): + """Computes size from a shape `Tensor`, statically if possible.""" + s = tensor_util.constant_value(shape) + if s is not None: + return [np.int32(np.prod(s))]*2 + return None, math_ops.reduce_prod(shape, name="size") + + # Ensure `event_shape_in` is compatible with `event_shape_out`. + event_size_in_, event_size_in = _get_size_from_shape( # pylint: disable=unbalanced-tuple-unpacking + event_shape_in) + event_size_out_, event_size_out = _get_size_from_shape( # pylint: disable=unbalanced-tuple-unpacking + event_shape_out) + + assertions = [] + if event_size_in_ is not None and event_size_out_ is not None: + if event_size_in_ != event_size_out_: + raise ValueError( + "Input `event_size` ({}) does not match output `event_size` ({}).". + format(event_size_in, event_size_out_)) + elif validate_args: + assertions.append(check_ops.assert_equal( + event_size_in, event_size_out, + message="Input/output `event_size`s do not match.")) + + return assertions + + def _reshape_helper(self, x, event_shape_in, event_shape_out): + """Reshape only the event_shape of an input `Tensor`.""" + + def _get_rank_from_shape(shape): + """Computes rank from a shape `Tensor`, statically if possible.""" + # Uses fact that rank is "shape of shape". + ndims = shape.shape.with_rank_at_least(1)[0].value + if ndims is not None: + return ndims, ndims + return None, array_ops.shape(shape)[0] + + event_ndims_in_, event_ndims_in = _get_rank_from_shape(event_shape_in) + + assertions = [] + # Ensure x.event_shape is compatible with event_shape_in. + if x.shape.ndims is not None: + x_ndims_, x_ndims = [x.shape.ndims]*2 + else: + x_ndims_, x_ndims = None, array_ops.rank(x) + + if (event_ndims_in_ is not None + and x_ndims_ is not None + and x.shape.with_rank_at_least(event_ndims_in_)[ + x_ndims_-event_ndims_in_:].is_fully_defined()): + x_event_shape_, x_event_shape = [ # pylint: disable=unbalanced-tuple-unpacking + np.int32(x.shape[x_ndims_-event_ndims_in_:])]*2 + else: + x_event_shape_, x_event_shape = ( + None, array_ops.shape(x)[x_ndims-event_ndims_in:]) + + event_shape_in_ = tensor_util.constant_value(event_shape_in) + + if x_event_shape_ is not None and event_shape_in_ is not None: + if not np.equal(x_event_shape_, event_shape_in_).all(): + raise ValueError( + "Input `event_shape` ({}) does not match `event_shape_in` ({}).". + format(x_event_shape_, event_shape_in_)) + elif self.validate_args: + assertions.append(check_ops.assert_equal( + x_event_shape, event_shape_in, + message="Input `event_shape` does not match `event_shape_in`.")) + + if assertions: + x = control_flow_ops.with_dependencies(assertions, x) + + # get the parts of shape(x) that will not change + sample_and_batch_shape = array_ops.shape(x) + + ndims = (x.shape.ndims if x.shape.ndims is not None + else array_ops.rank(x)) + sample_and_batch_shape = sample_and_batch_shape[ + :(ndims - math_ops.abs(event_ndims_in))] + + new_shape = array_ops.concat( + [sample_and_batch_shape, event_shape_out], axis=0) + + return array_ops.reshape(x, new_shape) + + def _forward(self, x): + with ops.control_dependencies(self._assertions): + return self._reshape_helper(x, + self._event_shape_in, + self._event_shape_out) + + def _inverse(self, y): + with ops.control_dependencies(self._assertions): + return self._reshape_helper(y, + self._event_shape_out, + self._event_shape_in) + + def _inverse_log_det_jacobian(self, y): + with ops.control_dependencies(self._assertions): + return constant_op.constant(0., dtype=y.dtype) + + def _forward_log_det_jacobian(self, x): + with ops.control_dependencies(self._assertions): + return constant_op.constant(0., dtype=x.dtype) + + def _forward_event_shape(self, input_shape): + self._event_shape_in_static.assert_is_compatible_with(input_shape) + return self._event_shape_out_static + + def _inverse_event_shape(self, output_shape): + self._event_shape_out_static.assert_is_compatible_with(output_shape) + return self._event_shape_in_static + + def _forward_event_shape_tensor(self, input_shape): + input_assertions = self._maybe_check_valid_shape( + input_shape, "input event shape", validate_args=self.validate_args) + input_assertions += self._maybe_check_matching_sizes( + input_shape, self._event_shape_out, + validate_args=self.validate_args) + + return control_flow_ops.with_dependencies( + input_assertions + self._assertions, self._event_shape_out) + + def _inverse_event_shape_tensor(self, output_shape): + + output_assertions = self._maybe_check_valid_shape( + output_shape, "output event shape", validate_args=self.validate_args) + output_assertions += self._maybe_check_matching_sizes( + output_shape, self._event_shape_in, validate_args=self.validate_args) + + return control_flow_ops.with_dependencies( + output_assertions + self._assertions, self._event_shape_in) diff --git a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py index f1b7bf468e92913e6d1d5dd965de9c3dc220f9ed..599c855cda434d9249187d5d154d50a8a8c49a6c 100644 --- a/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py +++ b/tensorflow/contrib/distributions/python/ops/conditional_transformed_distribution.py @@ -198,3 +198,19 @@ class ConditionalTransformedDistribution( distribution_kwargs = distribution_kwargs or {} x = self.bijector.inverse(y, **bijector_kwargs) return self.distribution.survival_function(x, **distribution_kwargs) + + @distribution_util.AppendDocstring(kwargs_dict=_condition_kwargs_dict) + def _quantile(self, value, bijector_kwargs=None, distribution_kwargs=None): + if self._is_maybe_event_override: + raise NotImplementedError("quantile is not implemented when overriding " + "event_shape") + if not self.bijector._is_injective: # pylint: disable=protected-access + raise NotImplementedError("quantile is not implemented when " + "bijector is not injective.") + bijector_kwargs = bijector_kwargs or {} + distribution_kwargs = distribution_kwargs or {} + # x_q is the "qth quantile" of X iff q = P[X <= x_q]. Now, since X = + # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)], + # implies the qth quantile of Y is g(x_q). + inv_cdf = self.distribution.quantile(value, **distribution_kwargs) + return self.bijector.forward(inv_cdf, **bijector_kwargs) diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py index 393c008242417f8a2bf44eed2d9b2e81800d34c7..6a74ca9a0ae1ad30081d21cc15a65be052a99e2a 100644 --- a/tensorflow/contrib/distributions/python/ops/independent.py +++ b/tensorflow/contrib/distributions/python/ops/independent.py @@ -45,24 +45,24 @@ class Independent(distribution_lib.Distribution): `p(x_1, ..., x_B) = p_1(x_1) * ... * p_B(x_B)` where `p_b(X_b)` is the probability of the `b`-th rv. More generally `B, E` can be arbitrary shapes. - Similarly, the `Independent` distribution specifies a distribution over - `[B, E]`-shaped events. It operates by reinterpreting the rightmost batch dims - as part of the event dimensions. The `reduce_batch_ndims` parameter controls - the number of batch dims which are absorbed as event dims; - `reduce_batch_ndims < len(batch_shape)`. For example, the `log_prob` function - entails a `reduce_sum` over the rightmost `reduce_batch_ndims` after calling - the base distribution's `log_prob`. In other words, since the batch - dimension(s) index independent distributions, the resultant multivariate will - have independent components. + Similarly, the `Independent` distribution specifies a distribution over `[B, + E]`-shaped events. It operates by reinterpreting the rightmost batch dims as + part of the event dimensions. The `reinterpreted_batch_ndims` parameter + controls the number of batch dims which are absorbed as event dims; + `reinterpreted_batch_ndims < len(batch_shape)`. For example, the `log_prob` + function entails a `reduce_sum` over the rightmost `reinterpreted_batch_ndims` + after calling the base distribution's `log_prob`. In other words, since the + batch dimension(s) index independent distributions, the resultant multivariate + will have independent components. #### Mathematical Details The probability function is, ```none - prob(x; reduce_batch_ndims) = tf.reduce_prod( + prob(x; reinterpreted_batch_ndims) = tf.reduce_prod( dist.prob(x), - axis=-1-range(reduce_batch_ndims)) + axis=-1-range(reinterpreted_batch_ndims)) ``` #### Examples @@ -73,7 +73,7 @@ class Independent(distribution_lib.Distribution): # Make independent distribution from a 2-batch Normal. ind = ds.Independent( distribution=ds.Normal(loc=[-1., 1], scale=[0.1, 0.5]), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) # All batch dims have been "absorbed" into event dims. ind.batch_shape # ==> [] @@ -84,7 +84,7 @@ class Independent(distribution_lib.Distribution): distribution=ds.MultivariateNormalDiag( loc=[[-1., 1], [1, -1]], scale_identity_multiplier=[1., 0.5]), - reduce_batch_ndims=1) + reinterpreted_batch_ndims=1) # All batch dims have been "absorbed" into event dims. ind.batch_shape # ==> [] @@ -94,14 +94,17 @@ class Independent(distribution_lib.Distribution): """ def __init__( - self, distribution, reduce_batch_ndims=1, validate_args=False, name=None): + self, distribution, reinterpreted_batch_ndims=None, + validate_args=False, name=None): """Construct a `Independent` distribution. Args: distribution: The base distribution instance to transform. Typically an instance of `Distribution`. - reduce_batch_ndims: Scalar, integer number of rightmost batch dims which - will be regard as event dims. + reinterpreted_batch_ndims: Scalar, integer number of rightmost batch dims + which will be regarded as event dims. When `None` all but the first + batch axis (batch axis 0) will be transferred to event dimensions + (analogous to `tf.layers.flatten`). validate_args: Python `bool`. Whether to validate input with asserts. If `validate_args` is `False`, and the inputs are invalid, correct behavior is not guaranteed. @@ -109,19 +112,25 @@ class Independent(distribution_lib.Distribution): Default value: `Independent + distribution.name`. Raises: - ValueError: if `reduce_batch_ndims` exceeds `distribution.batch_ndims` + ValueError: if `reinterpreted_batch_ndims` exceeds + `distribution.batch_ndims` """ parameters = locals() name = name or "Independent" + distribution.name self._distribution = distribution with ops.name_scope(name): - reduce_batch_ndims = ops.convert_to_tensor( - reduce_batch_ndims, dtype=dtypes.int32, name="reduce_batch_ndims") - self._reduce_batch_ndims = reduce_batch_ndims - self._static_reduce_batch_ndims = tensor_util.constant_value( - reduce_batch_ndims) - if self._static_reduce_batch_ndims is not None: - self._reduce_batch_ndims = self._static_reduce_batch_ndims + if reinterpreted_batch_ndims is None: + reinterpreted_batch_ndims = self._get_default_reinterpreted_batch_ndims( + distribution) + reinterpreted_batch_ndims = ops.convert_to_tensor( + reinterpreted_batch_ndims, + dtype=dtypes.int32, + name="reinterpreted_batch_ndims") + self._reinterpreted_batch_ndims = reinterpreted_batch_ndims + self._static_reinterpreted_batch_ndims = tensor_util.constant_value( + reinterpreted_batch_ndims) + if self._static_reinterpreted_batch_ndims is not None: + self._reinterpreted_batch_ndims = self._static_reinterpreted_batch_ndims super(Independent, self).__init__( dtype=self._distribution.dtype, reparameterization_type=self._distribution.reparameterization_type, @@ -129,19 +138,19 @@ class Independent(distribution_lib.Distribution): allow_nan_stats=self._distribution.allow_nan_stats, parameters=parameters, graph_parents=( - [reduce_batch_ndims] + + [reinterpreted_batch_ndims] + distribution._graph_parents), # pylint: disable=protected-access name=name) self._runtime_assertions = self._make_runtime_assertions( - distribution, reduce_batch_ndims, validate_args) + distribution, reinterpreted_batch_ndims, validate_args) @property def distribution(self): return self._distribution @property - def reduce_batch_ndims(self): - return self._reduce_batch_ndims + def reinterpreted_batch_ndims(self): + return self._reinterpreted_batch_ndims def _batch_shape_tensor(self): with ops.control_dependencies(self._runtime_assertions): @@ -149,13 +158,14 @@ class Independent(distribution_lib.Distribution): batch_ndims = (batch_shape.shape[0].value if batch_shape.shape.with_rank_at_least(1)[0].value else array_ops.shape(batch_shape)[0]) - return batch_shape[:batch_ndims - self.reduce_batch_ndims] + return batch_shape[:batch_ndims - self.reinterpreted_batch_ndims] def _batch_shape(self): batch_shape = self.distribution.batch_shape - if self._static_reduce_batch_ndims is None or batch_shape.ndims is None: + if (self._static_reinterpreted_batch_ndims is None + or batch_shape.ndims is None): return tensor_shape.TensorShape(None) - d = batch_shape.ndims - self._static_reduce_batch_ndims + d = batch_shape.ndims - self._static_reinterpreted_batch_ndims return batch_shape[:d] def _event_shape_tensor(self): @@ -165,15 +175,16 @@ class Independent(distribution_lib.Distribution): if batch_shape.shape.with_rank_at_least(1)[0].value else array_ops.shape(batch_shape)[0]) return array_ops.concat([ - batch_shape[batch_ndims - self.reduce_batch_ndims:], + batch_shape[batch_ndims - self.reinterpreted_batch_ndims:], self.distribution.event_shape_tensor(), ], axis=0) def _event_shape(self): batch_shape = self.distribution.batch_shape - if self._static_reduce_batch_ndims is None or batch_shape.ndims is None: + if (self._static_reinterpreted_batch_ndims is None + or batch_shape.ndims is None): return tensor_shape.TensorShape(None) - d = batch_shape.ndims - self._static_reduce_batch_ndims + d = batch_shape.ndims - self._static_reinterpreted_batch_ndims return batch_shape[d:].concatenate(self.distribution.event_shape) def _sample_n(self, n, seed): @@ -205,15 +216,16 @@ class Independent(distribution_lib.Distribution): return self.distribution.mode() def _make_runtime_assertions( - self, distribution, reduce_batch_ndims, validate_args): + self, distribution, reinterpreted_batch_ndims, validate_args): assertions = [] - static_reduce_batch_ndims = tensor_util.constant_value(reduce_batch_ndims) + static_reinterpreted_batch_ndims = tensor_util.constant_value( + reinterpreted_batch_ndims) batch_ndims = distribution.batch_shape.ndims - if batch_ndims is not None and static_reduce_batch_ndims is not None: - if static_reduce_batch_ndims > batch_ndims: - raise ValueError("reduce_batch_ndims({}) cannot exceed " + if batch_ndims is not None and static_reinterpreted_batch_ndims is not None: + if static_reinterpreted_batch_ndims > batch_ndims: + raise ValueError("reinterpreted_batch_ndims({}) cannot exceed " "distribution.batch_ndims({})".format( - static_reduce_batch_ndims, batch_ndims)) + static_reinterpreted_batch_ndims, batch_ndims)) elif validate_args: batch_shape = distribution.batch_shape_tensor() batch_ndims = ( @@ -221,13 +233,24 @@ class Independent(distribution_lib.Distribution): if batch_shape.shape.with_rank_at_least(1)[0].value is not None else array_ops.shape(batch_shape)[0]) assertions.append(check_ops.assert_less_equal( - reduce_batch_ndims, batch_ndims, - message="reduce_batch_ndims cannot exceed distribution.batch_ndims")) + reinterpreted_batch_ndims, batch_ndims, + message=("reinterpreted_batch_ndims cannot exceed " + "distribution.batch_ndims"))) return assertions def _reduce_sum(self, stat): - if self._static_reduce_batch_ndims is None: - range_ = array_ops.range(self._reduce_batch_ndims) + if self._static_reinterpreted_batch_ndims is None: + range_ = math_ops.range(self._reinterpreted_batch_ndims) else: - range_ = np.arange(self._static_reduce_batch_ndims) + range_ = np.arange(self._static_reinterpreted_batch_ndims) return math_ops.reduce_sum(stat, axis=-1-range_) + + def _get_default_reinterpreted_batch_ndims(self, distribution): + """Computes the default value for reinterpreted_batch_ndim __init__ arg.""" + ndims = distribution.batch_shape.ndims + if ndims is None: + which_maximum = math_ops.maximum + ndims = array_ops.shape(distribution.batch_shape_tensor())[0] + else: + which_maximum = np.maximum + return which_maximum(0, ndims - 1) diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py index 251c2dbdfa59135be92afca30de88f23b2a40b4d..300bdd5f6064a1cc9c336689ac4fae04338edb30 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py @@ -22,7 +22,6 @@ from tensorflow.contrib.distributions.python.ops import distribution_util from tensorflow.contrib.distributions.python.ops.bijectors import AffineLinearOperator from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops.distributions import kullback_leibler from tensorflow.python.ops.distributions import normal @@ -299,7 +298,10 @@ def _kl_brute_force(a, b, name=None): def squared_frobenius_norm(x): """Helper to make KL calculation slightly more readable.""" # http://mathworld.wolfram.com/FrobeniusNorm.html - return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1])) + # The gradient of KL[p,q] is not defined when p==q. The culprit is + # linalg_ops.norm, i.e., we cannot use the commented out code. + # return math_ops.square(linalg_ops.norm(x, ord="fro", axis=[-2, -1])) + return math_ops.reduce_sum(math_ops.square(x), axis=[-2, -1]) # TODO(b/35041439): See also b/35040945. Remove this function once LinOp # supports something like: diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py index e3d68f6b4c0d8837e42c8f0a20d8c711bb21c9d6..260dcc18f513d5440d3d39368539274c03faa72a 100644 --- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py +++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py @@ -121,6 +121,14 @@ class MultivariateNormalTriL( [-10, 0, 9]] # shape: [2, 3] mvn.prob(x).eval() # shape: [2] + # Instantiate a "learnable" MVN. + dims = 4 + with tf.variable_scope("model"): + mvn = ds.MultivariateNormalTriL( + loc=tf.get_variable(shape=[dims], dtype=tf.float32, name="mu"), + scale_tril=ds.fill_triangular( + tf.get_variable(shape=[dims * (dims + 1) / 2], + dtype=tf.float32, name="chol_Sigma"))) ``` """ diff --git a/tensorflow/contrib/distributions/python/ops/negative_binomial.py b/tensorflow/contrib/distributions/python/ops/negative_binomial.py index c8c396f6f80cf7f3228a75d279fff91ae15813ad..3a58df80da6c02b056f5e5a63bf41de5fc6d44a4 100644 --- a/tensorflow/contrib/distributions/python/ops/negative_binomial.py +++ b/tensorflow/contrib/distributions/python/ops/negative_binomial.py @@ -167,8 +167,8 @@ class NegativeBinomial(distribution.Distribution): def _log_unnormalized_prob(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) - return (self.total_count * math_ops.log1p(-self.probs) - + x * math_ops.log(self.probs)) + return (self.total_count * math_ops.log_sigmoid(-self.logits) + + x * math_ops.log_sigmoid(self.logits)) def _log_normalization(self, x): if self.validate_args: diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py index 65ee3a16d624822dd69f9dea1507b96703db12be..8a95038a3c8eccf8a75fea79d0a62f9883b4f13a 100644 --- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py +++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np +from tensorflow.contrib.distributions.python.ops import distribution_util from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -29,7 +30,6 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops.distributions import categorical as categorical_lib from tensorflow.python.ops.distributions import distribution as distribution_lib -from tensorflow.python.ops.distributions import util as distribution_util __all__ = [ @@ -55,8 +55,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): ``` where `lambda(z) = exp(sqrt(2) scale z + loc)` and the `prob,grid` terms - are from [Gauss--Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). Note that + are from [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). Note that the second line made the substitution: `z(l) = (log(l) - loc) / (sqrt(2) scale)` which implies `lambda(z)` [above] and `dl = sqrt(2) scale lambda(z) dz` @@ -65,8 +67,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): Poisson rate parameter. Unfortunately, the non-approximate distribution lacks an analytical probability density function (pdf). Therefore the `PoissonLogNormalQuadratureCompound` class implements an approximation based - on [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). + on [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). + Note: although the `PoissonLogNormalQuadratureCompound` is approximately the Poisson-LogNormal compound distribution, it is itself a valid distribution. Viz., it possesses a `sample`, `log_prob`, `mean`, `variance`, etc. which are @@ -76,9 +81,11 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): The `PoissonLogNormalQuadratureCompound` approximates a Poisson-LogNormal [compound distribution]( - https://en.wikipedia.org/wiki/Compound_probability_distribution). - Using variable-substitution and [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature) we can + https://en.wikipedia.org/wiki/Compound_probability_distribution). Using + variable-substitution and [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)) we can redefine the distribution to be a parameter-less convex combination of `deg` different Poisson samples. @@ -93,7 +100,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): : d=0, ..., deg-1 } ``` - where, [`grid, w = numpy.polynomial.hermite.hermgauss(deg)`]( + where, [e.g., `grid, w = numpy.polynomial.hermite.hermgauss(deg)`]( https://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.polynomial.hermite.hermgauss.html) and `prob = w / sqrt(pi)`. @@ -106,14 +113,15 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): pln = ds.PoissonLogNormalQuadratureCompound( loc=[0., -0.5], scale=1., - quadrature_polynomial_degree=10, + quadrature_grid_and_probs=( + np.polynomial.hermite.hermgauss(deg=10)), validate_args=True) """ def __init__(self, loc, scale, - quadrature_polynomial_degree=8, + quadrature_grid_and_probs=None, validate_args=False, allow_nan_stats=True, name="PoissonLogNormalQuadratureCompound"): @@ -124,8 +132,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): the LogNormal prior. scale: `float`-like (batch of) scalar `Tensor`; the scale parameter of the LogNormal prior. - quadrature_polynomial_degree: Python `int`-like scalar. - Default value: 8. + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. When `None`, defaults to: + `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -153,18 +163,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): "loc.dtype(\"{}\") does not match scale.dtype(\"{}\")".format( loc.dtype.name, scale.dtype.name)) - self._degree = quadrature_polynomial_degree - - grid, prob = np.polynomial.hermite.hermgauss( - deg=quadrature_polynomial_degree) - - # It should be that `sum(prob) == sqrt(pi)`, but self-normalization is - # more numerically stable. - prob = prob.astype(dtype.as_numpy_dtype) - prob /= np.linalg.norm(prob, ord=1) + grid, probs = distribution_util.process_quadrature_grid_and_probs( + quadrature_grid_and_probs, dtype, validate_args) + self._quadrature_grid = grid + self._quadrature_probs = probs + self._quadrature_size = distribution_util.dimension_size(probs, axis=0) self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(prob), + logits=math_ops.log(self._quadrature_probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -210,9 +216,14 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): return self._scale @property - def quadrature_polynomial_degree(self): - """Polynomial largest exponent used for Gauss-Hermite quadrature.""" - return self._degree + def quadrature_grid(self): + """Quadrature grid points.""" + return self._quadrature_grid + + @property + def quadrature_probs(self): + """Quadrature normalized weights.""" + return self._quadrature_probs def _batch_shape_tensor(self): return array_ops.broadcast_dynamic_shape( @@ -242,10 +253,10 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution): [batch_size])), seed=distribution_util.gen_new_seed( seed, "poisson_lognormal_quadrature_compound")) - # Stride `quadrature_polynomial_degree` for `batch_size` number of times. + # Stride `quadrature_size` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * self._degree, - delta=self._degree, + limit=batch_size * self._quadrature_size, + delta=self._quadrature_size, dtype=ids.dtype) ids += offset rate = array_ops.gather( diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py index 699cf45a73883a49d116fa70c81a4f9ecb36e598..b6becfa9fc93f189a1a7bf7b2a7af8dc1f2e9720 100644 --- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py +++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py @@ -130,7 +130,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): temperature, logits=None, probs=None, - dtype=dtypes.float32, + dtype=None, validate_args=False, allow_nan_stats=True, name="ExpRelaxedOneHotCategorical"): @@ -150,7 +150,8 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): `N - 1` dimensions index into a batch of independent distributions and the last dimension represents a vector of probabilities for each class. Only one of `logits` or `probs` should be passed in. - dtype: The type of the event samples (default: float32). + dtype: The type of the event samples (default: inferred from + logits/probs). validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -163,14 +164,21 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): """ parameters = locals() with ops.name_scope(name, values=[logits, probs, temperature]): + + self._logits, self._probs = distribution_util.get_logits_and_probs( + name=name, logits=logits, probs=probs, validate_args=validate_args, + multidimensional=True) + + if dtype is None: + dtype = self._logits.dtype + if not validate_args: + temperature = math_ops.cast(temperature, dtype) + with ops.control_dependencies([check_ops.assert_positive(temperature)] if validate_args else []): self._temperature = array_ops.identity(temperature, name="temperature") self._temperature_2d = array_ops.reshape(temperature, [-1, 1], name="temperature_2d") - self._logits, self._probs = distribution_util.get_logits_and_probs( - name=name, logits=logits, probs=probs, validate_args=validate_args, - multidimensional=True) logits_shape_static = self._logits.get_shape().with_rank_at_least(1) if logits_shape_static.ndims is not None: @@ -230,7 +238,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution): def _sample_n(self, n, seed=None): sample_shape = array_ops.concat([[n], array_ops.shape(self.logits)], 0) - logits = self.logits * array_ops.ones(sample_shape) + logits = self.logits * array_ops.ones(sample_shape, dtype=self.dtype) logits_2d = array_ops.reshape(logits, [-1, self.event_size]) # Uniform variates must be sampled from the open-interval `(0, 1)` rather # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` @@ -368,7 +376,7 @@ class RelaxedOneHotCategorical( temperature, logits=None, probs=None, - dtype=dtypes.float32, + dtype=None, validate_args=False, allow_nan_stats=True, name="RelaxedOneHotCategorical"): @@ -388,7 +396,8 @@ class RelaxedOneHotCategorical( dimensions index into a batch of independent distributions and the last dimension represents a vector of probabilities for each class. Only one of `logits` or `probs` should be passed in. - dtype: The type of the event samples (default: float32). + dtype: The type of the event samples (default: inferred from + logits/probs). validate_args: Unused in this distribution. allow_nan_stats: Python `bool`, default `True`. If `False`, raise an exception if a statistic (e.g. mean/mode/etc...) is undefined for any diff --git a/tensorflow/contrib/distributions/python/ops/test_util.py b/tensorflow/contrib/distributions/python/ops/test_util.py index da7d3907acb6ac1c6c01ff739aa19fcb95fbb53d..77f2a39273dc365a4ac202d846dd2bc364655c86 100644 --- a/tensorflow/contrib/distributions/python/ops/test_util.py +++ b/tensorflow/contrib/distributions/python/ops/test_util.py @@ -25,6 +25,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import histogram_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import variables as variables_ops __all__ = [ @@ -37,7 +38,7 @@ class DiscreteScalarDistributionTestHelpers(object): """DiscreteScalarDistributionTestHelpers.""" def run_test_sample_consistent_log_prob( - self, sess, dist, + self, sess_run_fn, dist, num_samples=int(1e5), num_threshold=int(1e3), seed=42, rtol=1e-2, atol=0.): """Tests that sample/log_prob are consistent with each other. @@ -50,7 +51,9 @@ class DiscreteScalarDistributionTestHelpers(object): are consistent. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. num_samples: Python `int` scalar indicating the number of Monte-Carlo @@ -86,7 +89,7 @@ class DiscreteScalarDistributionTestHelpers(object): probs = math_ops.exp(dist.log_prob(edges)) probs = array_ops.reshape(probs, shape=[-1, batch_size])[:, b] - [counts_, probs_] = sess.run([counts, probs]) + [counts_, probs_] = sess_run_fn([counts, probs]) valid = counts_ > num_threshold probs_ = probs_[valid] counts_ = counts_[valid] @@ -94,7 +97,7 @@ class DiscreteScalarDistributionTestHelpers(object): rtol=rtol, atol=atol) def run_test_sample_consistent_mean_variance( - self, sess, dist, + self, sess_run_fn, dist, num_samples=int(1e5), seed=24, rtol=1e-2, atol=0.): """Tests that sample/mean/variance are consistent with each other. @@ -103,7 +106,9 @@ class DiscreteScalarDistributionTestHelpers(object): to the same distribution. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. num_samples: Python `int` scalar indicating the number of Monte-Carlo @@ -129,7 +134,7 @@ class DiscreteScalarDistributionTestHelpers(object): mean_, variance_, stddev_ - ] = sess.run([ + ] = sess_run_fn([ sample_mean, sample_variance, sample_stddev, @@ -186,7 +191,7 @@ class VectorDistributionTestHelpers(object): def run_test_sample_consistent_log_prob( self, - sess, + sess_run_fn, dist, num_samples=int(1e5), radius=1., @@ -239,7 +244,9 @@ class VectorDistributionTestHelpers(object): https://en.wikipedia.org/wiki/Importance_sampling. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. The distribution must have non-zero probability of sampling every point @@ -279,33 +286,39 @@ class VectorDistributionTestHelpers(object): def monte_carlo_hypersphere_volume(dist, num_samples, radius, center): # https://en.wikipedia.org/wiki/Importance_sampling x = dist.sample(num_samples, seed=seed) + x = array_ops.identity(x) # Invalidate bijector cacheing. return math_ops.reduce_mean( math_ops.exp(-dist.log_prob(x)) * is_in_ball(x, radius, center), axis=0) - [ - batch_shape_, - actual_volume_, - sample_volume_, - ] = sess.run([ - dist.batch_shape_tensor(), - actual_hypersphere_volume( - dims=dist.event_shape_tensor()[0], - radius=radius), - monte_carlo_hypersphere_volume( - dist, - num_samples=num_samples, - radius=radius, - center=center), - ]) - + # Build graph. + with ops.name_scope( + "run_test_sample_consistent_log_prob", + values=[num_samples, radius, center] + dist._graph_parents): # pylint: disable=protected-access + batch_shape = dist.batch_shape_tensor() + actual_volume = actual_hypersphere_volume( + dims=dist.event_shape_tensor()[0], + radius=radius) + sample_volume = monte_carlo_hypersphere_volume( + dist, + num_samples=num_samples, + radius=radius, + center=center) + init_op = variables_ops.global_variables_initializer() + + # Execute graph. + sess_run_fn(init_op) + [batch_shape_, actual_volume_, sample_volume_] = sess_run_fn([ + batch_shape, actual_volume, sample_volume]) + + # Check results. self.assertAllClose(np.tile(actual_volume_, reps=batch_shape_), sample_volume_, rtol=rtol, atol=atol) def run_test_sample_consistent_mean_covariance( self, - sess, + sess_run_fn, dist, num_samples=int(1e5), seed=24, @@ -319,7 +332,9 @@ class VectorDistributionTestHelpers(object): to the same distribution. Args: - sess: Tensorflow session. + sess_run_fn: Python `callable` taking `list`-like of `Tensor`s and + returning a list of results after running one "step" of TensorFlow + computation, typically set to `sess.run`. dist: Distribution instance or object which implements `sample`, `log_prob`, `event_shape_tensor` and `batch_shape_tensor`. num_samples: Python `int` scalar indicating the number of Monte-Carlo @@ -353,7 +368,7 @@ class VectorDistributionTestHelpers(object): covariance_, variance_, stddev_ - ] = sess.run([ + ] = sess_run_fn([ sample_mean, sample_covariance, sample_variance, diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py index 438d628da481d387f74b40fab3de62349061668c..92043d6a08833888c36009261addca0d14949ea8 100644 --- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py +++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py @@ -73,8 +73,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): denotes matrix multiplication. However, the non-approximate distribution does not have an analytical probability density function (pdf). Therefore the `VectorDiffeomixture` class implements an approximation based on - [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature). I.e., in + [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)). I.e., in Note: although the `VectorDiffeomixture` is approximately the `SoftmaxNormal-Distribution` compound distribution, it is itself a valid distribution. It possesses a `sample`, `log_prob`, `mean`, `covariance` which @@ -109,8 +111,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): The `VectorDiffeomixture` approximates a SoftmaxNormal-mixed ("prior") [compound distribution]( https://en.wikipedia.org/wiki/Compound_probability_distribution). - Using variable-substitution and [Gauss-Hermite quadrature]( - https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature) we can + Using variable-substitution and [numerical quadrature]( + https://en.wikipedia.org/wiki/Numerical_integration) (default: + [Gauss--Hermite quadrature]( + https://en.wikipedia.org/wiki/Gauss%E2%80%93Hermite_quadrature)) we can redefine the distribution to be a parameter-less convex combination of `K` different affine combinations of a `d` iid samples from `distribution`. @@ -141,7 +145,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): and, ```none - grid, weight = np.polynomial.hermite.hermgauss(quadrature_polynomial_degree) + grid, weight = np.polynomial.hermite.hermgauss(quadrature_size) prob[k] = weight[k] / sqrt(pi) lambda[k; i] = sigmoid(mix_loc[k] + sqrt(2) mix_scale[k] grid[i]) ``` @@ -219,7 +223,7 @@ class VectorDiffeomixture(distribution_lib.Distribution): distribution, loc=None, scale=None, - quadrature_polynomial_degree=8, + quadrature_grid_and_probs=None, validate_args=False, allow_nan_stats=True, name="VectorDiffeomixture"): @@ -248,7 +252,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): `k`-th element represents the `scale` used for the `k`-th affine transformation. `LinearOperator`s must have shape `[B1, ..., Bb, d, d]`, `b >= 0`, i.e., characterizes `b`-batches of `d x d` matrices - quadrature_polynomial_degree: Python `int`-like scalar. + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. When `None`, defaults to: + `np.polynomial.hermite.hermgauss(deg=8)`. validate_args: Python `bool`, default `False`. When `True` distribution parameters are checked for validity despite possibly degrading runtime performance. When `False` invalid inputs may silently render incorrect @@ -262,7 +269,8 @@ class VectorDiffeomixture(distribution_lib.Distribution): Raises: ValueError: if `not scale or len(scale) < 2`. ValueError: if `len(loc) != len(scale)` - ValueError: if `quadrature_polynomial_degree < 1`. + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` ValueError: if `validate_args` and any not scale.is_positive_definite. TypeError: if any scale.dtype != scale[0].dtype. TypeError: if any loc.dtype != scale[0].dtype. @@ -307,12 +315,6 @@ class VectorDiffeomixture(distribution_lib.Distribution): name="endpoint_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip(loc, scale))] - if quadrature_polynomial_degree < 1: - raise ValueError("quadrature_polynomial_degree={} " - "is not at least 1".format( - quadrature_polynomial_degree)) - self._degree = quadrature_polynomial_degree - # TODO(jvdillon): Remove once we support k-mixtures. # We make this assertion here because otherwise `grid` would need to be a # vector not a scalar. @@ -320,17 +322,17 @@ class VectorDiffeomixture(distribution_lib.Distribution): raise NotImplementedError("Currently only bimixtures are supported; " "len(scale)={} is not 2.".format(len(scale))) - grid, prob = np.polynomial.hermite.hermgauss( - deg=quadrature_polynomial_degree) - grid = grid.astype(dtype.as_numpy_dtype) - prob = prob.astype(dtype.as_numpy_dtype) - prob /= np.linalg.norm(prob, ord=1) + grid, probs = distribution_util.process_quadrature_grid_and_probs( + quadrature_grid_and_probs, dtype, validate_args) + self._quadrature_grid = grid + self._quadrature_probs = probs + self._quadrature_size = distribution_util.dimension_size(probs, axis=0) # Note: by creating the logits as `log(prob)` we ensure that # `self.mixture_distribution.logits` is equivalent to # `math_ops.log(self.mixture_distribution.probs)`. self._mixture_distribution = categorical_lib.Categorical( - logits=np.log(prob), + logits=math_ops.log(probs), validate_args=validate_args, allow_nan_stats=allow_nan_stats) @@ -357,10 +359,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): validate_args=validate_args, name="interpolated_affine_{}".format(k)) for k, (loc_, scale_) in enumerate(zip( - interpolate_loc(quadrature_polynomial_degree, + interpolate_loc(self._quadrature_size, self._interpolate_weight, loc), - interpolate_scale(quadrature_polynomial_degree, + interpolate_scale(self._quadrature_size, self._interpolate_weight, scale)))] @@ -416,9 +418,14 @@ class VectorDiffeomixture(distribution_lib.Distribution): return self._interpolated_affine @property - def quadrature_polynomial_degree(self): - """Polynomial largest exponent used for Gauss-Hermite quadrature.""" - return self._degree + def quadrature_grid(self): + """Quadrature grid points.""" + return self._quadrature_grid + + @property + def quadrature_probs(self): + """Quadrature normalized weights.""" + return self._quadrature_probs def _batch_shape_tensor(self): return self._batch_shape_ @@ -454,10 +461,10 @@ class VectorDiffeomixture(distribution_lib.Distribution): seed=distribution_util.gen_new_seed( seed, "vector_diffeomixture")) - # Stride `self._degree` for `batch_size` number of times. + # Stride `quadrature_size` for `batch_size` number of times. offset = math_ops.range(start=0, - limit=batch_size * self._degree, - delta=self._degree, + limit=batch_size * self._quadrature_size, + delta=self._quadrature_size, dtype=ids.dtype) weight = array_ops.gather( diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md index a4a3af08cf27d20147539cd0dde1f5e3a9d46918..ae4b07799f5c123b68529443a1765fbfbac05492 100644 --- a/tensorflow/contrib/eager/README.md +++ b/tensorflow/contrib/eager/README.md @@ -1,15 +1,78 @@ -TensorFlow has many kernels for doing (deep) learning and data manipulation. -There are typically assembled into computational graphs which can run -efficiently in a variety of environments. +# TensorFlow Eager Execution -We are exploring an alternative interaction, where kernels are invoked -immediately and call this "eager execution". We are hoping to retain the -benefits of graphs while improving usability with benefits like: +> *WARNING*: This is a preview/pre-alpha version. The API and performance +> characteristics are subject to change. -- Immediate error messages and easier debugging -- Flexibility to use Python datastructures and control flow -- Reduced boilerplate +Eager execution is an experimental interface to TensorFlow that provides an +imperative programming style (à la [NumPy](http://www.numpy.org)). When you +enable eager execution, TensorFlow operations execute immediately; you do not +execute a pre-constructed graph with +[`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session). -Eager execution is under active development. -There are not many developer-facing materials yet, but stay tuned for updates -in this directory. +For example, consider a simple computation in TensorFlow: + +```python +x = tf.placeholder(tf.float32, shape=[1, 1]) +m = tf.matmul(x, x) + +with tf.Session() as sess: + print(sess.run(m, feed_dict={x: [[2.]]})) + +# Will print [[4.]] +``` + +Eager execution makes this much simpler: + +```python +x = [[2.]] +m = tf.matmul(x, x) + +print(m) +``` + +## Caveats + +This feature is in early stages and work remains to be done in terms of smooth +support for distributed and multi-GPU training and CPU performance. + +- [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Acomp%3Aeager) +- Feedback is welcome, please consider + [filing an issue](https://github.com/tensorflow/tensorflow/issues/new) to provide it. + +## Installation + +Since eager execution is not yet part of a TensorFlow release, using it requires +either [building from source](https://www.tensorflow.org/install/install_sources) +or the latest nightly builds. The nightly builds are available as: + +- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and + +- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images. + +For example, to run the latest nightly docker image: + +```sh +# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker +nvidia-docker pull tensorflow/tensorflow:nightly-gpu +nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu + +# If you do not have a GPU, use the CPU-only image +docker pull tensorflow/tensorflow:nightly +docker run -it -p 8888:8888 tensorflow/tensorflow:nightly +``` + +And then visit http://localhost:8888 in your browser for a Jupyter notebook +environment. Try out the notebooks below. + +## Documentation + +For an introduction to eager execution in TensorFlow, see: + +- [User Guide](python/g3doc/guide.md) +- Notebook: [Basic Usage](python/examples/notebooks/1_basics.ipynb) +- Notebook: [Gradients](python/examples/notebooks/2_gradients.ipynb) +- Notebook: [Importing Data](python/examples/notebooks/3_datasets.ipynb) + +## Changelog + +- 2017/10/31: Initial preview release. diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD index 0c61630aa8f79e3efd25584478547abd99f30285..2b84bc2e9b7453fac99ea2becc328ca854cf555d 100644 --- a/tensorflow/contrib/eager/python/BUILD +++ b/tensorflow/contrib/eager/python/BUILD @@ -18,6 +18,8 @@ py_library( ":saver", ":summary_writer", "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:numerics", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:util", "//tensorflow/python/eager:backprop", @@ -35,9 +37,11 @@ cuda_py_test( additional_deps = [ ":tfe", "//tensorflow/python:array_ops", + "//tensorflow/python:metrics", "//tensorflow/python:math_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:platform_test", + "//tensorflow/python:summary", ], ) @@ -47,8 +51,10 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ + "//tensorflow/python:array_ops", "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python/data/util:nest", "//tensorflow/python/eager:context", @@ -61,10 +67,12 @@ py_test( srcs_version = "PY2AND3", deps = [ ":datasets", - "//tensorflow/contrib/data", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:script_ops", + "//tensorflow/python/data", "//tensorflow/python/eager:test", - "//third_party/py/numpy", ], ) @@ -73,7 +81,11 @@ py_library( srcs = ["saver.py"], srcs_version = "PY2AND3", deps = [ + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:training", + "//tensorflow/python/eager:context", ], ) @@ -86,7 +98,7 @@ cuda_py_test( "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python/eager:graph_callable", - "//tensorflow/python:platform_test", + "//tensorflow/python/eager:test", "//tensorflow/python:variables", ], ) @@ -97,12 +109,14 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/contrib/summary:gen_summary_ops", - "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:init_ops", "//tensorflow/python:resource_variable_ops", "//tensorflow/python:state_ops", "//tensorflow/python:summary_op_util", - "//tensorflow/python:training", + "//tensorflow/python:variable_scope", "//tensorflow/python/eager:context", ], ) @@ -129,14 +143,17 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ + "//tensorflow/contrib/summary:summary_ops", "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:init_ops", - "//tensorflow/python:layers_base", "//tensorflow/python:math_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:function", ], ) @@ -146,6 +163,15 @@ py_test( srcs_version = "PY2AND3", deps = [ ":metrics", + "//tensorflow/contrib/summary:summary_ops", + "//tensorflow/contrib/summary:summary_test_util", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:lib", + "//tensorflow/python:platform", + "//tensorflow/python:training", + "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], ) @@ -160,6 +186,13 @@ py_library( deps = [ ":datasets", ":metrics", + "//tensorflow/contrib/summary:summary_ops", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:function", + "@six_archive//:six", ], ) @@ -170,7 +203,12 @@ py_test( deps = [ ":evaluator", ":metrics", + "//tensorflow/contrib/summary:summary_test_util", + "//tensorflow/python:framework_ops", + "//tensorflow/python:training", + "//tensorflow/python:variables", "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/eager:context", "//tensorflow/python/eager:test", ], ) @@ -181,9 +219,9 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - "//tensorflow/python:framework_ops", "//tensorflow/python:layers_base", "//tensorflow/python:variable_scope", + "//tensorflow/python/estimator:util", ], ) @@ -194,7 +232,12 @@ py_test( deps = [ ":network", "//tensorflow/python:constant_op", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:layers", + "//tensorflow/python:math_ops", + "//tensorflow/python:nn_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:variable_scope", "//tensorflow/python/eager:test", ], ) diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py index fb9fabd6c1b48b9e3a4572d4eb8f6546f2f17c43..98e6983658aed77277d87915ff26a8c676224503 100644 --- a/tensorflow/contrib/eager/python/datasets.py +++ b/tensorflow/contrib/eager/python/datasets.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Support for tf.contrib.data when eager execution is enabled.""" +"""Iteration over tf.data.Datasets when eager execution is enabled.""" from __future__ import absolute_import from __future__ import division @@ -24,6 +24,7 @@ from tensorflow.python.data.util import nest from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import resource_variable_ops @@ -40,20 +41,23 @@ def _iterator_shared_name(): class Iterator(object): - """An iterator producing tf.Tensor objects from a tf.contrib.data.Dataset.""" + """An iterator producing tf.Tensor objects from a tf.data.Dataset.""" def __init__(self, dataset): """Creates a new iterator over the given dataset. For example: ```python - dataset = tf.contrib.data.Dataset.range(4) + dataset = tf.data.Dataset.range(4) for x in Iterator(dataset): print(x) ``` + Tensors produced will be placed on the device on which this iterator object + was created. + Args: - dataset: A `tf.contrib.data.Dataset` object. + dataset: A `tf.data.Dataset` object. Raises: RuntimeError: When invoked without eager execution enabled. @@ -61,8 +65,10 @@ class Iterator(object): if not context.in_eager_mode(): raise RuntimeError( - "{} objects only make sense when eager execution is enabled".format( - type(self))) + "{} objects can only be used when eager execution is enabled, use " + "tf.data.Dataset.make_iterator or " + "tf.data.Dataset.make_one_shot_iterator for graph construction". + format(type(self))) with ops.device("/device:CPU:0"): ds_variant = dataset._as_variant_tensor() # pylint: disable=protected-access self._output_types = dataset.output_types @@ -74,12 +80,10 @@ class Iterator(object): output_types=self._flat_output_types, output_shapes=self._flat_output_shapes) gen_dataset_ops.make_iterator(ds_variant, self._resource) - - def __del__(self): - if self._resource is not None: - with ops.device("/device:CPU:0"): - resource_variable_ops.destroy_resource_op(self._resource) - self._resource = None + # Delete the resource when this object is deleted + self._resource_deleter = resource_variable_ops.EagerResourceDeleter( + handle=self._resource, handle_device="/device:CPU:0") + self._device = context.context().device_name def __iter__(self): return self @@ -98,6 +102,11 @@ class Iterator(object): self._resource, output_types=self._flat_output_types, output_shapes=self._flat_output_shapes) - return nest.pack_sequence_as(self._output_types, ret) except errors.OutOfRangeError: raise StopIteration + # Copies tensors from CPU to the current device if necessary. + # TODO(rohanj): This should be replaced by the mechanism to have the + # runtime's threads copy tensors to the destination device. + with ops.device(self._device): + ret = [array_ops.identity(x) for x in ret] + return nest.pack_sequence_as(self._output_types, ret) diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py index 076c92e73f7c2a1ebc6dbeac940a8307adc16414..c924d81c9d85e638e4f35f260664c0ee7d03257e 100644 --- a/tensorflow/contrib/eager/python/datasets_test.py +++ b/tensorflow/contrib/eager/python/datasets_test.py @@ -16,10 +16,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.contrib.data import Dataset from tensorflow.contrib.eager.python import datasets +from tensorflow.python.data import Dataset from tensorflow.python.eager import test from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import script_ops @@ -81,6 +82,13 @@ class IteratorTest(test.TestCase): got = [x.numpy() for x in datasets.Iterator(ds)] self.assertAllEqual([[1], [2], [3], [4]], got) + def testTensorsPlacedOnDevice(self): + ds = Dataset.from_tensors([0., 1.]) + with ops.device(test.gpu_device_name()): + x = datasets.Iterator(ds).next() + x = math_ops.add(x, x) + self.assertAllEqual([0., 2.], x.numpy()) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/eager/python/evaluator.py b/tensorflow/contrib/eager/python/evaluator.py index d757e976eeafa36ec5e870cfde0c620a204d7440..bd0ab02ecf7ae6025e08dde1c3ddc634db9255c1 100644 --- a/tensorflow/contrib/eager/python/evaluator.py +++ b/tensorflow/contrib/eager/python/evaluator.py @@ -22,6 +22,12 @@ import six from tensorflow.contrib.eager.python import datasets from tensorflow.contrib.eager.python import metrics +from tensorflow.contrib.summary import summary_ops +from tensorflow.python.eager import context +from tensorflow.python.eager import function +from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import ops +from tensorflow.python.ops import control_flow_ops class Evaluator(object): @@ -31,13 +37,13 @@ class Evaluator(object): evaluator = my_model.evaluator() # or MyEvaluator(my_model) for example_batch in ...: evaluator(example_batch) - results = evaluator.all_metric_results(optional_summary_writer) + results = evaluator.all_metric_results(optional_summary_logdir) Or, if you are getting your examples from a tf.data.Dataset, you can use the evaluate_on_dataset() method. Implementers of Evaluators should - (a) Call `add_metric()` and/or `add_evaluator()` in __init__(). + (a) Call `track_metric()` and/or `track_evaluator()` in __init__(). (b) Override the `call()` method. It will be passed the output of the model's `eval_data()` method, and should call its contained metrics (treating them as callables) and any child Evaluators (using their @@ -51,14 +57,69 @@ class Evaluator(object): self._model = model self._metrics = {} self._evaluators = {} + if context.in_graph_mode(): + self.call = function.defun(self.call) # ---- API for users ---- def __call__(self, *args, **kwargs): - """Update metrics with a minibatch of input examples.""" + """Update metrics with a minibatch of input examples. + + Args: + *args: + **kwargs: Arguments representing an input mini-batch of examples to + pass to self.model.eval_data(). + + Returns: + The op to execute or None if executing eagerly. + """ return self.call(self._model.eval_data(*args, **kwargs)) - def all_metric_results(self): # TODO(josh11b): Add optional summary_writer. - """Returns dict mapping metric name -> value.""" + def init_variables(self): + """Return an op for initializing all contained uninitialized variables. + + Only for graph execution. Should be called after variables are created + in the first execution of __call__(). + + Returns: + An op. + + Raises: + RuntimeError: if eager execution is enabled. + + @compatibility(eager) + Only for graph execution. + @end_compatibility + """ + if context.in_eager_mode(): + raise RuntimeError("Evaluator.init_variables() not needed when " + "eager execution is enabled.") + return control_flow_ops.group([m.init_variables() for _, m in self.metrics]) + + def all_metric_results(self, summary_logdir=None): + """Computes results for all contained metrics. + + Args: + summary_logdir: An optional string. If specified, metric results + will be written as summaries to this directory. + + Returns: + A `dict` mapping string names to tensors. + """ + if summary_logdir is None: + with summary_ops.never_record_summaries(): + return self._all_metric_results() + else: + def f(): + with summary_ops.create_summary_file_writer( + summary_logdir).as_default(), summary_ops.always_record_summaries(): + return self._all_metric_results() + if context.in_eager_mode(): + return f() + else: + return function.defun(f)() + + def _all_metric_results(self): + """Implementation of `all_metric_results` in the summary context.""" results = {} for name, metric in six.iteritems(self._metrics): results[name] = metric.result() @@ -68,15 +129,99 @@ class Evaluator(object): return results def evaluate_on_dataset(self, dataset, *args, **kwargs): - """Convenience method for performing an eval on a Dataset.""" + """Convenience method for performing an eval on a Dataset. + + Args: + dataset: Dataset object with the input data to evaluate on. + *args: + **kwargs: Optional additional arguments to __call__(), except + `summary_logdir`: if specified, metrics will be written as summaries + to this directory. + + Returns: + @compatibility(eager) + When eager execution is enabled, this returns the result of performing + an evaluation as a dictionary. With graph execution, this returns a tuple + (init_op, call_op, results_op) which may be executed using this code: + ```python + sess.run(init_op) + try: + while True: + sess.run(call_op) + except tf.errors.OutOfRangeError: + pass + return sess.run(results_op) # A dictionary + + # equivalently: + return evaluator.run_evaluation(init_op, call_op, results_op, sess=sess) + ``` + @end_compatibility + """ + summary_logdir = kwargs.pop("summary_logdir", None) + if context.in_graph_mode(): + call_op = self.__call__(dataset.make_one_shot_iterator().get_next(), + *args, **kwargs) + init_op = self.init_variables() + results_op = self.all_metric_results(summary_logdir) + return (init_op, call_op, results_op) + # Eager case for example in datasets.Iterator(dataset): self.__call__(example, *args, **kwargs) - # TODO(josh11b): Add optional summary_writer. - return self.all_metric_results() + return self.all_metric_results(summary_logdir) + + @staticmethod + def run_evaluation(init_op, call_op, results_op, sess=None): + """Convenience method for running the ops returned by evaluate_on_dataset. + + Args: + init_op: An op that initializes/resets evaluation state. + call_op: An op that updates evaluation state on a mini-batch of examples. + Must generate an tf.errors.OutOfRangeError when done. + results_op: A dictionary of tensors that compute the final evaluation + results from the evaulation state. + sess: The Session to run the evaluation in. Defaults to the default + Session. + + Returns: + A dictionary of values, parallel to results_op. + + Raises: + RuntimeError: if eager execution is enabled. + + @compatibility(eager) + Only for graph execution. + @end_compatibility + """ + if context.in_eager_mode(): + raise RuntimeError("Evaluator.run_evaluation() not supported when " + "eager execution is enabled.") + sess = sess or ops.get_default_session() + sess.run(init_op) + try: + while True: + sess.run(call_op) + except errors_impl.OutOfRangeError: + pass + return sess.run(results_op) # ---- To be implemented by descendants --- def call(self, eval_data): - """Update metrics using the output of self.model.""" + """Update metrics using the output of self.model. + + Note: This function is executed as a graph function in graph mode. + This means: + a) Operations on the same resource are executed in textual order. + This should make it easier to do things like add the updated + value of a variable to another, for example. + b) You don't need to worry about collecting the update ops to execute. + All update ops added to the graph by this function will be executed. + As a result, code should generally work the same way with graph or + eager execution. + + Args: + eval_data: The output of self.model.eval_data() on a mini-batch of + examples. + """ raise NotImplementedError("Evaluators must define a call member function.") # ---- For use by descendants --- @@ -84,10 +229,11 @@ class Evaluator(object): def model(self): return self._model - def add_metric(self, metric): + def track_metric(self, metric): """Add a Metric to be tracked. - Rule: metrics can only be in one `Evaluator`. + Metrics can only be tracked by one `Evaluator`. Metrics must be + tracked or they will not appear in `all_metric_results()`. Args: metric: A `Metric` object. @@ -98,14 +244,15 @@ class Evaluator(object): Raises: RuntimeError: If called before __init__. TypeError: If `metric` is not of the correct type. - ValueError: If there is a name collision between Metrics. + ValueError: If there is a name collision between Metrics or `metric` + has already been added to another `Evaluator`. """ if not hasattr(self, "_metrics"): raise RuntimeError( "Need to call Evaluator.__init__ before adding metrics") if not isinstance(metric, metrics.Metric): raise TypeError( - "Evaluator.add_metric() passed type %s, not a tfe.metrics.Metric" % + "Evaluator.track_metric() passed type %s, not a tfe.metrics.Metric" % (type(metric),)) if metric.name in self._metrics: if metric is self._metrics[metric.name]: @@ -113,10 +260,16 @@ class Evaluator(object): raise ValueError( "Attempt to add two Metrics with the name '%s' to the same Evaluator " "'%s'" % (metric.name, self.name)) + # pylint: disable=protected-access + if hasattr(metric, "_added_to_an_evaluator"): + raise ValueError("Metric %s already added to Evaluator %s" % + (metric.name, metric._added_to_an_evaluator)) + metric._added_to_an_evaluator = self.__class__.__name__ + # pylint: enable=protected-access self._metrics[metric.name] = metric return metric - def add_evaluator(self, prefix, evaluator): + def track_evaluator(self, prefix, evaluator): """Add a contained `Evaluator`. This is for delegating to another `Evaluator`, e.g. for when you have a @@ -141,7 +294,7 @@ class Evaluator(object): "Need to call Evaluator.__init__ before adding evaluators") if not isinstance(evaluator, Evaluator): raise TypeError( - "Evaluator.add_evaluator() passed type %s, not a tfe.Evaluator." % + "Evaluator.track_evaluator() passed type %s, not a tfe.Evaluator." % (type(evaluator),)) if prefix in self._evaluators: if evaluator is self._evaluators[prefix]: @@ -162,11 +315,12 @@ class Evaluator(object): @property def metrics(self): + """Returns a list of (prefix, metric) pairs.""" m = [] for metric in six.itervalues(self._metrics): - m.append(metric) - for evaluator in six.itervalues(self._evaluators): - m += evaluator.metrics + m.append(("", metric)) + for prefix, evaluator in six.iteritems(self._evaluators): + m += [(prefix + "/" + p, m) for p, m in evaluator.metrics] return m @@ -196,8 +350,8 @@ class SparseSoftmaxEvaluator(Evaluator): super(SparseSoftmaxEvaluator, self).__init__(model) # TODO(josh11b): Expand this to include everything from the standard # SparseSoftmax Head. - self.avg_loss = self.add_metric(metrics.Mean("Avg_Loss")) - self.accuracy = self.add_metric(metrics.Accuracy()) + self.avg_loss = self.track_metric(metrics.Mean("Avg Loss")) + self.accuracy = self.track_metric(metrics.Accuracy()) self.loss_key = loss_key self.label_key = label_key self.predicted_class_key = predicted_class_key diff --git a/tensorflow/contrib/eager/python/evaluator_test.py b/tensorflow/contrib/eager/python/evaluator_test.py index 099e10e2307b2e3c406ccf847fc8ee2bce9ce407..02f82cb216983accc7bc2dfa20cbb1ee0b8d8d26 100644 --- a/tensorflow/contrib/eager/python/evaluator_test.py +++ b/tensorflow/contrib/eager/python/evaluator_test.py @@ -18,10 +18,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import tempfile + from tensorflow.contrib.eager.python import evaluator + from tensorflow.contrib.eager.python import metrics +from tensorflow.contrib.summary import summary_test_util from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.eager import context from tensorflow.python.eager import test +from tensorflow.python.framework import ops +from tensorflow.python.ops import variables +from tensorflow.python.training import training_util class IdentityModel(object): @@ -40,7 +48,7 @@ class SimpleEvaluator(evaluator.Evaluator): def __init__(self, model): super(SimpleEvaluator, self).__init__(model) - self.mean = self.add_metric(metrics.Mean("mean")) + self.mean = self.track_metric(metrics.Mean("mean")) def call(self, eval_data): self.mean(eval_data) @@ -50,8 +58,8 @@ class DelegatingEvaluator(evaluator.Evaluator): def __init__(self, model): super(DelegatingEvaluator, self).__init__(model) - self.sub = self.add_evaluator("inner", SimpleEvaluator(model)) - self.mean = self.add_metric(metrics.Mean("outer-mean")) + self.sub = self.track_evaluator("inner", SimpleEvaluator(model)) + self.mean = self.track_metric(metrics.Mean("outer-mean")) def call(self, eval_data): # Keys here come from PrefixLModel, which adds "l_". @@ -70,6 +78,19 @@ class EvaluatorTest(test.TestCase): self.assertEqual(set(["mean"]), set(results.keys())) self.assertEqual(6.0, results["mean"].numpy()) + def testWriteSummaries(self): + e = SimpleEvaluator(IdentityModel()) + e(3.0) + e([5.0, 7.0, 9.0]) + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + + e.all_metric_results(logdir) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 6.0) + def testComposition(self): e = DelegatingEvaluator(PrefixLModel()) e({"inner": 2.0, "outer": 100.0}) @@ -86,15 +107,39 @@ class EvaluatorTest(test.TestCase): for v in e.metric_variables: p = v.name.split("/")[0] prefix_count[p] = prefix_count.get(p, 0) + 1 - self.assertEqual({"outer-mean": 2, "mean": 2}, prefix_count) + self.assertEqual({"outer_mean": 2, "mean": 2}, prefix_count) - def testDataset(self): + def testDatasetEager(self): e = SimpleEvaluator(IdentityModel()) ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) results = e.evaluate_on_dataset(ds) self.assertEqual(set(["mean"]), set(results.keys())) self.assertEqual(6.0, results["mean"].numpy()) + def testDatasetGraph(self): + with context.graph_mode(), ops.Graph().as_default(), self.test_session(): + e = SimpleEvaluator(IdentityModel()) + ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) + init_op, call_op, results_op = e.evaluate_on_dataset(ds) + results = e.run_evaluation(init_op, call_op, results_op) + self.assertEqual(set(["mean"]), set(results.keys())) + self.assertEqual(6.0, results["mean"]) + + def testWriteSummariesGraph(self): + with context.graph_mode(), ops.Graph().as_default(), self.test_session(): + e = SimpleEvaluator(IdentityModel()) + ds = dataset_ops.Dataset.from_tensor_slices([3.0, 5.0, 7.0, 9.0]) + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + init_op, call_op, results_op = e.evaluate_on_dataset( + ds, summary_logdir=logdir) + variables.global_variables_initializer().run() + e.run_evaluation(init_op, call_op, results_op) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 6.0) + def testModelProperty(self): m = IdentityModel() e = SimpleEvaluator(m) @@ -102,8 +147,34 @@ class EvaluatorTest(test.TestCase): def testMetricsProperty(self): e = DelegatingEvaluator(PrefixLModel()) - names = set([m.name for m in e.metrics]) - self.assertEqual(set(["outer-mean", "mean"]), names) + names = set([(p, m.name) for p, m in e.metrics]) + self.assertEqual(set([("", "outer-mean"), ("inner/", "mean")]), names) + + def testSharedMetric(self): + + class MetricArgEvaluator(evaluator.Evaluator): + + def __init__(self, model, m): + super(MetricArgEvaluator, self).__init__(model) + self.m = self.track_metric(m) + + metric = metrics.Mean("mean") + model = IdentityModel() + e = MetricArgEvaluator(model, metric) + with self.assertRaisesRegexp(ValueError, "already added"): + MetricArgEvaluator(model, metric) + del e + + def testMetricTrackedTwice(self): + + class MetricTwiceEvaluator(evaluator.Evaluator): + + def __init__(self, model): + super(MetricTwiceEvaluator, self).__init__(model) + self.m = self.track_metric(metrics.Mean("mean")) + self.track_metric(self.m) # okay to track same metric again + + MetricTwiceEvaluator(IdentityModel()) class SparseSoftmaxEvaluatorTest(test.TestCase): @@ -115,8 +186,8 @@ class SparseSoftmaxEvaluatorTest(test.TestCase): e.label_key: [1, 2, 3], e.predicted_class_key: [1, 1, 3]}) results = e.all_metric_results() - self.assertEqual(set(["Avg_Loss", "Accuracy"]), set(results.keys())) - self.assertEqual(2.0, results["Avg_Loss"].numpy()) + self.assertEqual(set(["Avg Loss", "Accuracy"]), set(results.keys())) + self.assertEqual(2.0, results["Avg Loss"].numpy()) self.assertEqual(0.75, results["Accuracy"].numpy()) diff --git a/tensorflow/contrib/eager/python/examples/BUILD b/tensorflow/contrib/eager/python/examples/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..aa21a6ab994acf929890ecebc07a86cf7ebf97db --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/BUILD @@ -0,0 +1,15 @@ +# TensorFlow code for training gradient boosted trees. +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +py_library( + name = "examples_pip", + deps = [ + "//tensorflow/contrib/eager/python/examples/linear_regression", + "//tensorflow/contrib/eager/python/examples/mnist", + "//tensorflow/contrib/eager/python/examples/resnet50", + "//tensorflow/contrib/eager/python/examples/rnn_colorbot", + "//tensorflow/contrib/eager/python/examples/rnn_ptb", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..bab7ad0c701b2110fda9a8d27792fd361a5fc1c0 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD @@ -0,0 +1,25 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "linear_regression", + srcs = ["linear_regression.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + ], +) + +cuda_py_test( + name = "linear_regression_test", + size = "small", + srcs = ["linear_regression_test.py"], + additional_deps = [ + ":linear_regression", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..d0130ebd118dbaff4f0161c8b2528764c6103e02 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py @@ -0,0 +1,157 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""TensorFlow Eager Execution Example: Linear Regression. + +This example shows how to use TensorFlow Eager Execution to fit a simple linear +regression model using some synthesized data. Specifically, it illustrates how +to define the forward path of the linear model and the loss function, as well +as how to obtain the gradients of the loss function with respect to the +variables and update the variables with the gradients. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import sys + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe + + +class LinearModel(tfe.Network): + """A TensorFlow linear regression model. + + Uses TensorFlow's eager execution. + + For those familiar with TensorFlow graphs, notice the absence of + `tf.Session`. The `forward()` method here immediately executes and + returns output values. The `loss()` method immediately compares the + output of `forward()` with the target adn returns the MSE loss value. + The `fit()` performs gradient-descent training on the model's weights + and bias. + """ + + def __init__(self): + """Constructs a LinearModel object.""" + super(LinearModel, self).__init__() + self._hidden_layer = self.track_layer(tf.layers.Dense(1)) + + def call(self, xs): + """Invoke the linear model. + + Args: + xs: input features, as a tensor of size [batch_size, ndims]. + + Returns: + ys: the predictions of the linear mode, as a tensor of size [batch_size] + """ + return self._hidden_layer(xs) + + +def fit(model, dataset, optimizer, verbose=False, logdir=None): + """Fit the linear-regression model. + + Args: + model: The LinearModel to fit. + dataset: The tf.data.Dataset to use for training data. + optimizer: The TensorFlow Optimizer object to be used. + verbose: If true, will print out loss values at every iteration. + logdir: The directory in which summaries will be written for TensorBoard + (optional). + """ + + # The loss function to optimize. + def mean_square_loss(xs, ys): + return tf.reduce_mean(tf.square(model(xs) - ys)) + + loss_and_grads = tfe.implicit_value_and_gradients(mean_square_loss) + + tf.train.get_or_create_global_step() + if logdir: + # Support for TensorBoard summaries. Once training has started, use: + # tensorboard --logdir= + summary_writer = tf.contrib.summary.create_summary_file_writer(logdir) + + # Training loop. + for i, (xs, ys) in enumerate(tfe.Iterator(dataset)): + loss, grads = loss_and_grads(xs, ys) + if verbose: + print("Iteration %d: loss = %s" % (i, loss.numpy())) + + optimizer.apply_gradients(grads, global_step=tf.train.get_global_step()) + + if logdir: + with summary_writer.as_default(): + with tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar("loss", loss) + + +def synthetic_dataset(w, b, noise_level, batch_size, num_batches): + """tf.data.Dataset that yields synthetic data for linear regression.""" + + # w is a matrix with shape [N, M] + # b is a vector with shape [M] + # So: + # - Generate x's as vectors with shape [batch_size N] + # - y = tf.matmul(x, W) + b + noise + def batch(_): + x = tf.random_normal([batch_size, tf.shape(w)[0]]) + y = tf.matmul(x, w) + b + noise_level * tf.random_normal([]) + return x, y + + with tf.device("/device:CPU:0"): + return tf.data.Dataset.range(num_batches).map(batch) + + +def main(_): + tfe.enable_eager_execution() + # Ground-truth constants. + true_w = [[-2.0], [4.0], [1.0]] + true_b = [0.5] + noise_level = 0.01 + + # Training constants. + batch_size = 64 + learning_rate = 0.1 + + print("True w: %s" % true_w) + print("True b: %s\n" % true_b) + + model = LinearModel() + dataset = synthetic_dataset(true_w, true_b, noise_level, batch_size, 20) + + device = "gpu:0" if tfe.num_gpus() else "cpu:0" + print("Using device: %s" % device) + with tf.device(device): + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + fit(model, dataset, optimizer, verbose=True, logdir=FLAGS.logdir) + + print("\nAfter training: w = %s" % model.variables[0].numpy()) + print("\nAfter training: b = %s" % model.variables[1].numpy()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--logdir", + type=str, + default=None, + help="logdir in which TensorBoard summaries will be written (optional).") + FLAGS, unparsed = parser.parse_known_args() + + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py new file mode 100644 index 0000000000000000000000000000000000000000..39e7aabd7be04ba36a786a4c08d0df6c2ce916d0 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression_test.py @@ -0,0 +1,119 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Unit tests for linear regression example under TensorFlow eager execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import glob +import os +import shutil +import tempfile +import time + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.contrib.eager.python.examples.linear_regression import linear_regression + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() > 0 else "/device:CPU:0" + + +class LinearRegressionTest(tf.test.TestCase): + + def setUp(self): + super(LinearRegressionTest, self).setUp() + self._tmp_logdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tmp_logdir) + super(LinearRegressionTest, self).tearDown() + + def testSyntheticDataset(self): + true_w = tf.random_uniform([3, 1]) + true_b = [1.0] + batch_size = 10 + num_batches = 2 + noise_level = 0. + dataset = linear_regression.synthetic_dataset(true_w, true_b, noise_level, + batch_size, num_batches) + + it = tfe.Iterator(dataset) + for _ in range(2): + (xs, ys) = it.next() + self.assertEqual((batch_size, 3), xs.shape) + self.assertEqual((batch_size, 1), ys.shape) + self.assertEqual(tf.float32, xs.dtype) + self.assertEqual(tf.float32, ys.dtype) + with self.assertRaises(StopIteration): + it.next() + + def testLinearRegression(self): + true_w = [[1.0], [-0.5], [2.0]] + true_b = [1.0] + + model = linear_regression.LinearModel() + dataset = linear_regression.synthetic_dataset( + true_w, true_b, noise_level=0., batch_size=64, num_batches=40) + + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) + linear_regression.fit(model, dataset, optimizer, logdir=self._tmp_logdir) + + self.assertAllClose(true_w, model.variables[0].numpy(), rtol=1e-2) + self.assertAllClose(true_b, model.variables[1].numpy(), rtol=1e-2) + self.assertTrue(glob.glob(os.path.join(self._tmp_logdir, "events.out.*"))) + + +class EagerLinearRegressionBenchmark(tf.test.Benchmark): + + def benchmarkEagerLinearRegression(self): + num_batches = 200 + batch_size = 64 + dataset = linear_regression.synthetic_dataset( + w=tf.random_uniform([3, 1]), + b=tf.random_uniform([1]), + noise_level=0.01, + batch_size=batch_size, + num_batches=num_batches) + burn_in_dataset = dataset.take(10) + + model = linear_regression.LinearModel() + + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1) + + # Perform burn-in. + linear_regression.fit(model, burn_in_dataset, optimizer) + + start_time = time.time() + linear_regression.fit(model, dataset, optimizer) + wall_time = time.time() - start_time + + examples_per_sec = num_batches * batch_size / wall_time + self.report_benchmark( + name="eager_train_%s" % + ("gpu" if tfe.num_gpus() > 0 else "cpu"), + iters=num_batches, + extras={"examples_per_sec": examples_per_sec}, + wall_time=wall_time) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/mnist/BUILD b/tensorflow/contrib/eager/python/examples/mnist/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..c61ec2dbae60a782c0e6589701554b045dcb92ae --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/BUILD @@ -0,0 +1,36 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "mnist", + srcs = ["mnist.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow/examples/tutorials/mnist:input_data", + ], +) + +cuda_py_test( + name = "mnist_test", + srcs = ["mnist_test.py"], + additional_deps = [ + ":mnist", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) + +cuda_py_test( + name = "mnist_graph_test", + srcs = ["mnist_graph_test.py"], + additional_deps = [ + ":mnist", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/mnist/README.md b/tensorflow/contrib/eager/python/examples/mnist/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e987996b88ccf54a322749aadec4f9840760a90f --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/README.md @@ -0,0 +1,10 @@ +Classification model for the MNIST dataset using eager execution. + +To run: + +``` +python mnist.py +``` + +`mnist_graph_test.py` demonstrates that the same code that is executed eagerly +in `mnist.py` is used to construct a TensorFlow graph. diff --git a/tensorflow/contrib/eager/python/examples/mnist/mnist.py b/tensorflow/contrib/eager/python/examples/mnist/mnist.py new file mode 100644 index 0000000000000000000000000000000000000000..ae01bac0b560e15f655c883da4ccc1944c07232c --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/mnist.py @@ -0,0 +1,270 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A deep MNIST classifier using convolutional layers. + +Sample usage: + python mnist.py --help +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import functools +import os +import sys +import time + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.examples.tutorials.mnist import input_data + +FLAGS = None + + +class MNISTModel(tfe.Network): + """MNIST Network. + + Network structure is equivalent to: + https://github.com/tensorflow/tensorflow/blob/r1.4/tensorflow/examples/tutorials/mnist/mnist_deep.py + and + https://github.com/tensorflow/models/blob/master/tutorials/image/mnist/convolutional.py + + But written using the tf.layers API. + """ + + def __init__(self, data_format): + """Creates a model for classifying a hand-written digit. + + Args: + data_format: Either 'channels_first' or 'channels_last'. + 'channels_first' is typically faster on GPUs while 'channels_last' is + typically faster on CPUs. See + https://www.tensorflow.org/performance/performance_guide#data_formats + """ + super(MNISTModel, self).__init__(name='') + if data_format == 'channels_first': + self._input_shape = [-1, 1, 28, 28] + else: + assert data_format == 'channels_last' + self._input_shape = [-1, 28, 28, 1] + self.conv1 = self.track_layer( + tf.layers.Conv2D(32, 5, data_format=data_format, activation=tf.nn.relu)) + self.conv2 = self.track_layer( + tf.layers.Conv2D(64, 5, data_format=data_format, activation=tf.nn.relu)) + self.fc1 = self.track_layer(tf.layers.Dense(1024, activation=tf.nn.relu)) + self.fc2 = self.track_layer(tf.layers.Dense(10)) + self.dropout = self.track_layer(tf.layers.Dropout(0.5)) + self.max_pool2d = self.track_layer( + tf.layers.MaxPooling2D( + (2, 2), (2, 2), padding='SAME', data_format=data_format)) + + def call(self, inputs, training): + """Computes labels from inputs. + + Users should invoke __call__ to run the network, which delegates to this + method (and not call this method directly). + + Args: + inputs: A batch of images as a Tensor with shape [batch_size, 784]. + training: True if invoked in the context of training (causing dropout to + be applied). False otherwise. + + Returns: + A Tensor with shape [batch_size, 10] containing the predicted logits + for each image in the batch, for each of the 10 classes. + """ + + x = tf.reshape(inputs, self._input_shape) + x = self.conv1(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.max_pool2d(x) + x = tf.layers.flatten(x) + x = self.fc1(x) + if training: + x = self.dropout(x) + x = self.fc2(x) + return x + + +def loss(predictions, labels): + return tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + logits=predictions, labels=labels)) + + +def compute_accuracy(predictions, labels): + return tf.reduce_sum( + tf.cast( + tf.equal( + tf.argmax(predictions, axis=1, + output_type=tf.int64), + tf.argmax(labels, axis=1, + output_type=tf.int64)), + dtype=tf.float32)) / float(predictions.shape[0].value) + + +def train_one_epoch(model, optimizer, dataset, log_interval=None): + """Trains model on `dataset` using `optimizer`.""" + + tf.train.get_or_create_global_step() + + def model_loss(labels, images): + prediction = model(images, training=True) + loss_value = loss(prediction, labels) + tf.contrib.summary.scalar('loss', loss_value) + tf.contrib.summary.scalar('accuracy', + compute_accuracy(prediction, labels)) + return loss_value + + for (batch, (images, labels)) in enumerate(tfe.Iterator(dataset)): + with tf.contrib.summary.record_summaries_every_n_global_steps(10): + batch_model_loss = functools.partial(model_loss, labels, images) + optimizer.minimize( + batch_model_loss, global_step=tf.train.get_global_step()) + if log_interval and batch % log_interval == 0: + print('Batch #%d\tLoss: %.6f' % (batch, batch_model_loss())) + + +def test(model, dataset): + """Perform an evaluation of `model` on the examples from `dataset`.""" + avg_loss = tfe.metrics.Mean('loss') + accuracy = tfe.metrics.Accuracy('accuracy') + + for (images, labels) in tfe.Iterator(dataset): + predictions = model(images, training=False) + avg_loss(loss(predictions, labels)) + accuracy(tf.argmax(predictions, axis=1, output_type=tf.int64), + tf.argmax(labels, axis=1, output_type=tf.int64)) + print('Test set: Average loss: %.4f, Accuracy: %4f%%\n' % + (avg_loss.result(), 100 * accuracy.result())) + with tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar('loss', avg_loss.result()) + tf.contrib.summary.scalar('accuracy', accuracy.result()) + + +def load_data(data_dir): + """Returns training and test tf.data.Dataset objects.""" + data = input_data.read_data_sets(data_dir, one_hot=True) + train_ds = tf.data.Dataset.from_tensor_slices((data.train.images, + data.train.labels)) + test_ds = tf.data.Dataset.from_tensors((data.test.images, data.test.labels)) + return (train_ds, test_ds) + + +def main(_): + tfe.enable_eager_execution() + + (device, data_format) = ('/gpu:0', 'channels_first') + if FLAGS.no_gpu or tfe.num_gpus() <= 0: + (device, data_format) = ('/cpu:0', 'channels_last') + print('Using device %s, and data format %s.' % (device, data_format)) + + # Load the datasets + (train_ds, test_ds) = load_data(FLAGS.data_dir) + train_ds = train_ds.shuffle(60000).batch(FLAGS.batch_size) + + # Create the model and optimizer + model = MNISTModel(data_format) + optimizer = tf.train.MomentumOptimizer(FLAGS.lr, FLAGS.momentum) + + if FLAGS.output_dir: + train_dir = os.path.join(FLAGS.output_dir, 'train') + test_dir = os.path.join(FLAGS.output_dir, 'eval') + tf.gfile.MakeDirs(FLAGS.output_dir) + else: + train_dir = None + test_dir = None + summary_writer = tf.contrib.summary.create_summary_file_writer( + train_dir, flush_secs=10) + test_summary_writer = tf.contrib.summary.create_summary_file_writer( + test_dir, flush_secs=10, name='test') + checkpoint_prefix = os.path.join(FLAGS.checkpoint_dir, 'ckpt') + + with tf.device(device): + for epoch in range(1, 11): + with tfe.restore_variables_on_create( + tf.train.latest_checkpoint(FLAGS.checkpoint_dir)): + global_step = tf.train.get_or_create_global_step() + start = time.time() + with summary_writer.as_default(): + train_one_epoch(model, optimizer, train_ds, FLAGS.log_interval) + end = time.time() + print('\nTrain time for epoch #%d (global step %d): %f' % ( + epoch, global_step.numpy(), end - start)) + with test_summary_writer.as_default(): + test(model, test_ds) + all_variables = ( + model.variables + + tfe.get_optimizer_variables(optimizer) + + [global_step]) + tfe.Saver(all_variables).save( + checkpoint_prefix, global_step=global_step) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + '--data-dir', + type=str, + default='/tmp/tensorflow/mnist/input_data', + help='Directory for storing input data') + parser.add_argument( + '--batch-size', + type=int, + default=64, + metavar='N', + help='input batch size for training (default: 64)') + parser.add_argument( + '--log-interval', + type=int, + default=10, + metavar='N', + help='how many batches to wait before logging training status') + parser.add_argument( + '--output_dir', + type=str, + default=None, + metavar='N', + help='Directory to write TensorBoard summaries') + parser.add_argument( + '--checkpoint_dir', + type=str, + default='/tmp/tensorflow/mnist/checkpoints/', + metavar='N', + help='Directory to save checkpoints in (once per epoch)') + parser.add_argument( + '--lr', + type=float, + default=0.01, + metavar='LR', + help='learning rate (default: 0.01)') + parser.add_argument( + '--momentum', + type=float, + default=0.5, + metavar='M', + help='SGD momentum (default: 0.5)') + parser.add_argument( + '--no-gpu', + action='store_true', + default=False, + help='disables GPU usage even if a GPU is available') + + FLAGS, unparsed = parser.parse_known_args() + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py b/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py new file mode 100644 index 0000000000000000000000000000000000000000..1af26553120b34d4682b17b1c29c81dc65e421d4 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py @@ -0,0 +1,65 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np +import tensorflow as tf +from tensorflow.contrib.eager.python.examples.mnist import mnist + + +def data_format(): + return "channels_first" if tf.test.is_gpu_available() else "channels_last" + + +class MNISTGraphTest(tf.test.TestCase): + + def testTrainGraph(self): + # The MNISTModel class can be executed eagerly (as in mnist.py and + # mnist_test.py) and also be used to construct a TensorFlow graph, which is + # then trained in a session. + with tf.Graph().as_default(): + # Generate some random data. + batch_size = 64 + images = np.random.randn(batch_size, 784).astype(np.float32) + digits = np.random.randint(low=0, high=10, size=batch_size) + labels = np.zeros((batch_size, 10)) + labels[np.arange(batch_size), digits] = 1. + + # Create a model, optimizer, and dataset as would be done + # for eager execution as well. + model = mnist.MNISTModel(data_format()) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + dataset = tf.data.Dataset.from_tensors((images, labels)) + + # Define the loss tensor (as opposed to a loss function when + # using eager execution). + (images, labels) = dataset.make_one_shot_iterator().get_next() + predictions = model(images, training=True) + loss = mnist.loss(predictions, labels) + + train_op = optimizer.minimize(loss) + init = tf.global_variables_initializer() + with tf.Session() as sess: + # Variables have to be initialized in the session. + sess.run(init) + # Train using the optimizer. + sess.run(train_op) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/mnist/mnist_test.py b/tensorflow/contrib/eager/python/examples/mnist/mnist_test.py new file mode 100644 index 0000000000000000000000000000000000000000..205709fe2edd3c260c30a84b624e322e120edf8e --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/mnist/mnist_test.py @@ -0,0 +1,62 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.contrib.eager.python.examples.mnist import mnist + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" + + +def data_format(): + return "channels_first" if tfe.num_gpus() else "channels_last" + + +def random_dataset(): + batch_size = 64 + images = tf.random_normal([batch_size, 784]) + digits = tf.random_uniform([batch_size], minval=0, maxval=10, dtype=tf.int32) + labels = tf.one_hot(digits, 10) + return tf.data.Dataset.from_tensors((images, labels)) + + +class MNISTTest(tf.test.TestCase): + + def testTrainOneEpoch(self): + model = mnist.MNISTModel(data_format()) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + dataset = random_dataset() + with tf.device(device()): + tf.train.get_or_create_global_step() + mnist.train_one_epoch(model, optimizer, dataset) + + def testTest(self): + model = mnist.MNISTModel(data_format()) + dataset = random_dataset() + with tf.device(device()): + tf.train.get_or_create_global_step() + mnist.test(model, dataset) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..01616f2e7dbab8084153e6554ce0e64c13f5d710 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/1_basics.ipynb @@ -0,0 +1,529 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U9i2Dsh-ziXr" + }, + "source": [ + "# Eager Execution Tutorial: Basics\n", + "\n", + "This notebook introduces the basics of using TensorFlow's eager execution capabilities. It covers concepts such as:\n", + "\n", + "* Importing required packages\n", + "* Enabling eager execution\n", + "* Creating and using TensorFlow Tensors and Variables\n", + "* Using TensorFlow interactively\n", + "* Using GPUs with eager execution enabled\n", + "\n", + "This notebook does *not* cover modeling topics, such as gradients." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z1JcS5iBXMRO" + }, + "source": [ + "# Step 1: Import Eager\n", + "\n", + "The key imports for eager execution are the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RlIWhyeLoYnG" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "import tensorflow.contrib.eager as tfe" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H9UySOPLXdaw" + }, + "source": [ + "# Step 2: Enable eager execution\n", + "\n", + "All future TensorFlow calls will execute the\n", + "underlying TensorFlow ops immediately:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "WPTUfGq6kJ5w" + }, + "outputs": [], + "source": [ + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "twBfWd5xyu_d" + }, + "source": [ + "# Step 3: Interactively Use TensorFlow!\n", + "\n", + "Now you can call TensorFlow functions and get results, immediately! No more `tf.Sessions`!\n", + "\n", + "TensorFlow will automatically wrap native Python types for you with operator overloading for TensorFlow Tensors." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ngUe237Wt48W" + }, + "outputs": [], + "source": [ + "print(tf.add(1, 2))\n", + "print(tf.add([1, 2], [3, 4]))\n", + "print(tf.square(5))\n", + "print(tf.reduce_sum([1, 2, 3]))\n", + "print(tf.encode_base64(\"hello world\"))\n", + "print(\"\")\n", + "\n", + "x = tf.constant(2)\n", + "y = tf.constant(3)\n", + "print(x * y + 1)\n", + "\n", + "# Most TensorFlow ops are directly usable with eager execution, giving\n", + "# results immediately.\n", + "print(tf.contrib.signal.hamming_window(x * y + 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IDY4WsYRhP81" + }, + "source": [ + "Numpy arrays are supported, too:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "lCUWzso6mbqR" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "ones = np.ones([3, 3])\n", + "\n", + "print(\"numpy 3x3 matrix of 1s:\")\n", + "print(ones)\n", + "print(\"\")\n", + "\n", + "print(\"Multiplied by 42:\")\n", + "print(tf.multiply(ones, 42))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "PBNP8yTRfu_X" + }, + "source": [ + "# Step 4: Define and Print TensorFlow Variables\n", + "\n", + "To define TensorFlow variables, use the `get_variable()` function as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "3Twf_Rw-gQFM" + }, + "outputs": [], + "source": [ + "x = tf.get_variable(name=\"x\", shape=[], dtype=tf.float32, initializer=tf.zeros_initializer)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "45G7094TxsMb" + }, + "source": [ + "## Printing TensorFlow Variables" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "UJBJeZ5XxuwA" + }, + "outputs": [], + "source": [ + "# This does NOT print the Variable's actual value:\n", + "print(\"Printing a TensorFlow Variable:\")\n", + "print(x)\n", + "print(\"\")\n", + "\n", + "# A TensorFlow variable represents a reference to a tensor.\n", + "# The `read_value()` method provides access to the current value of the\n", + "# variable. Tensorflow Variables are automatically initialized according to the\n", + "# semantics defined in tf.get_variable().\n", + "print(\"Printing a TensorFlow Variable's value using .read_value():\")\n", + "print(x.read_value())\n", + "print(\"\")\n", + "\n", + "print(\"Printing a TensorFlow Variable's value using .read_value().numpy():\")\n", + "print(x.read_value().numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "2njjWHcTpBEn" + }, + "source": [ + "## Changing a TensorFlow Variable's value\n", + "\n", + "To change a TensorFlow Variable's value, use its `.assign()` or `.assign_add()` method:" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "v3wr6Erbo_hB" + }, + "outputs": [], + "source": [ + "x.assign(42)\n", + "print(x.read_value())\n", + "\n", + "x.assign_add(3)\n", + "print(x.read_value())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "uhtynjHVpTB5" + }, + "source": [ + "## Use a Variable just like any other Tensor" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "7PbktdnHoehR" + }, + "outputs": [], + "source": [ + "print(x + 3)\n", + "\n", + "# This code will broadcast the value across the list of numbers:\n", + "print(x * [1, 2, 4])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GVChqwlwy1SI" + }, + "source": [ + "# Step 5: Debug Errors with Instant Feedback\n", + "\n", + "TensorFlow's eager execution helps you identify and debug runtime issues through interactive exploration of code snippets.\n", + "\n", + "Below, we'll define a length-4 vector, and attempt two `tf.slice()` operations,\n", + "one being legal and the other being illegal, leading to a runtime error that is\n", + "raised immediately." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "23ap04N0v4k0" + }, + "outputs": [], + "source": [ + "vector = tf.constant([10.0, 20.0, 30.0, 40.0])" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "FCUMsIYxxRRa" + }, + "outputs": [], + "source": [ + "# Works, because the values of `begin` and `size` (the 2nd and 3rd input\n", + "# arguments) are within the bound of `vector`.\n", + "print(tf.slice(vector, [1], [3]))" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "T8me2oCNxpFp" + }, + "outputs": [], + "source": [ + "# The following does NOT work, because the value of `size` (the 3rd\n", + "# argument) causes the indices to go out of the bounds of `vector`. The\n", + "# error is raised immediately.\n", + "try:\n", + " print(tf.slice(vector, [1], [4]))\n", + "except tf.OpError as e:\n", + " print(\"Caught error: %s\" % e)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "irxJhAgar84v" + }, + "source": [ + "# Step 6: Using the GPU\n", + "\n", + "You can place Tensors on the GPU by calling a Tensor's `.gpu()` method.\n", + "\n", + "The first operation executing on the GPU may be slow as TensorFlow initializes. Subsequent uses will be much faster." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "7J4N9baqaKCL" + }, + "outputs": [], + "source": [ + "# The example code from here on will work only if your notebook\n", + "# is running on a machine with a functional CUDA GPU. The following\n", + "# line checks that.\n", + "is_gpu_available = tfe.num_gpus() \u003e 0\n", + "\n", + "# Create some Tensors\n", + "SIZE = 1000\n", + "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", + "\n", + "if is_gpu_available:\n", + " gpu_tensor = cpu_tensor.gpu()" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "4E-2n7VbzY1n" + }, + "outputs": [], + "source": [ + "# Time a CPU-based matrix multiplication\n", + "\n", + "print(\"Time to conduct matmul on CPU:\")\n", + "%time tf.matmul(cpu_tensor, cpu_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "vbSFW-T5zhZF" + }, + "outputs": [], + "source": [ + "# Time GPU-based matrix multiplications.\n", + "\n", + "if is_gpu_available:\n", + " # First use of the GPU will be slow:\n", + " print(\"Time to conduct first matmul on GPU:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)\n", + " print()\n", + "\n", + " # Subsequent uses are much faster:\n", + " print(\"Time to conduct second matmul on GPU:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "E5pIOe3Rz7iW" + }, + "outputs": [], + "source": [ + "# Second timing demo for GPUs, after it has been used once:\n", + "\n", + "cpu_tensor = tf.random_normal([SIZE, SIZE])\n", + "print(\"Time to conduct CPU matmul:\")\n", + "%time tf.matmul(cpu_tensor, cpu_tensor)\n", + "print()\n", + "\n", + "if is_gpu_available:\n", + " gpu_tensor = cpu_tensor.gpu()\n", + " print(\"Time to conduct GPU matmul:\")\n", + " %time tf.matmul(gpu_tensor, gpu_tensor)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "name": "Eager Execution Tutorial: Basics", + "provenance": [ + { + "file_id": "0B0kLcpwLFwKEVm9XNkFueGk4bTg", + "timestamp": 1504118841551 + } + ], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..3b7e2cd435e7f34cb950545a9fe5ee6eafefde7e --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/2_gradients.ipynb @@ -0,0 +1,864 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "vDJ4XzMqodTy" + }, + "source": [ + "# Eager Execution: Working with Gradients\n", + "\n", + "This notebook demonstrates:\n", + "\n", + "* How to get gradients using TensorFlow's eager execution capabilities\n", + "* How to apply the gradients so you can update your variables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "GQJysDM__Qb0" + }, + "source": [ + "# Setup: Import eager and enable eager execution.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "OiMPZStlibBv" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "import tensorflow.contrib.eager as tfe\n", + "\n", + "# Enable eager execution.\n", + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "1CLWJl0QliB0" + }, + "source": [ + "# Fitting a Simple Linear Model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "-39gouo7mtgu" + }, + "source": [ + "## Step 1: Synthesize some data\n", + "\n", + "To demonstrate fitting a model with TensorFlow's eager execution, we'll fit a linear model to some synthesized data (which includes some noise).\n", + "\n", + "In the code, we use the variable names `w` and `b` to represent the single weight and bias we'll use to fit our model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "rQsdCg9PfIL-" + }, + "outputs": [], + "source": [ + "# The constants we'll try to fit our variables to:\n", + "true_w = 3\n", + "true_b = 2\n", + "\n", + "NUM_EXAMPLES = 1000\n", + "\n", + "# Our inputs:\n", + "inputs = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", + "\n", + "# Our labels, with noise:\n", + "noise = tf.random_normal(shape=[NUM_EXAMPLES, 1])\n", + "labels = inputs * true_w + true_b + noise" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 360, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 127, + "status": "ok", + "timestamp": 1505502830690, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "O4lsC4ckAcar", + "outputId": "2f760690-cafb-4777-b970-91d839f99faf" + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAesAAAFXCAYAAACC+2avAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt8VPWd99+TK7kykxtJQIebqZfaqogtrhKNa1ooEKl9\nCrpVn9ZNW6x9VWsbCi7aVUt01NZ9tq21KVZlFey2YkQNohhj3QWK2liCF5RIBCc3yEwmIZnMTOY8\nf/zmzJwzSSBAYibh+369eIU5c87vXLh8zvdu0TRNQxAEQRCEmCVurC9AEARBEISjI2ItCIIgCDGO\niLUgCIIgxDgi1oIgCIIQ44hYC4IgCEKMI2ItCIIgCDHOiIj16tWrufjii1m8eHF4269//Wvmz5/P\n0qVLWbp0Ka+//vpInEoQBEEQTjksI1Fn/eabb5KWlkZFRQWbN28GlFinpaXx7W9/+6QvUhAEQRBO\nZUbEsr7wwgvJzMwcsF36rQiCIAjCyTOqMesnn3ySsrIybr/9drq6ukbzVIIgCIIwYRk1sb722mt5\n5ZVXqK6uJicnh8rKytE6lSAIgiBMaEZNrLOysrBYLAB885vfZPfu3cc8RtzmgiAIgjCQhJFaKFpo\n29vbyc3NBeDll1+mqKjomGtYLBba2yeuuzw3N0Pubxwzke9vIt8byP2Nd06F+zsWIyLWt912Gzt3\n7sTtdnPZZZfxwx/+kJ07d/Lee+8RFxfH1KlTueuuu0biVIIgCIJwyjEiYv3ggw8O2Hb11VePxNKC\nIAiCcMojHcwEQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBOHXo6HCzcmUt\nTU2Z2O2dOBwl2GzWsb6smEfEWhAEQfjMWLmylurq6wAL9fUasJ6qqqVjfVkxj7jBBUEQhM+MpqZM\nwBL6ZAl9Fo6FiLUgCILwmWG3dwJa6JOG3e4Zy8sZN4gbXBAEQfjMcDhKgPWhmLUHh+Pysb6kcYGI\ntSAIgvCZYbNZJUZ9AogbXBAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFr\nQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfE\nWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYR\nsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhxRKwFQRAEIcYRsRYEQRCEGEfEWhAEQRBiHBFrQRAEQYhx\nRKwFQRAEIcYRsRYEQRCEGCdhrC9AEARBODE6OtysXFmL02mjsLADh6MEm806rGOamjKx2zuHdYww\n9oyIWK9evZrXXnuN7OxsNm/eDEBnZye33norn376KdOmTeOhhx4iIyNjJE4nCIIgACtX1lJdfR1g\nATRgPVVVS037RIuzz9dDTc33AQv19YMfI8QeI+IG//rXv866detM237/+98zb948XnrpJb70pS/x\nyCOPjMSpBEEQhBBNTZkooQawhD6b0QW9vv4qqquvZ/v27mMeI8QeIyLWF154IZmZ5j/wbdu2sXSp\neltbunQpr7zyykicShAEQQhht3eiLGoADbvdM2CfaEGH7GMeI8Qeoxaz7ujoICcnB4Dc3FxcLtdo\nnUoQBOGUxOEoAdaHYtYuHI7LAbPru61tD1AM2AAXkyY5sVr/CBxi3rwMHI5FY3cDwrCJuQSz3NyJ\nHdeW+xvfTOT7m8j3BhPz/uLi+klOTgQgOTmBnJwMsrIyuPnm5w2x7DKmTbuPgoJzaG7ew8GDt6PH\nuDMyNpKdncFNNz3Pxx+nM2NGFw8/vJCsrNhLOJuIf37Hw6iJdXZ2NocOHSInJ4f29naysrKGdVx7\ne9doXdKYk5ubIfc3jpnI9zeR7w0m7v2Vlz8XFuVduzT6+lSy2N69KRhd3zk5Z/LCC5dRWtrPwYOR\n7Xv3pnDjjYOvEUtM1D8/neG8iIxYnbWmaabPJSUlPPPMMwBs2rSJK664YqROJQiCIDB0gtlQsezB\ntg8nSU0Ye0bEsr7tttvYuXMnbrebyy67jB/+8Id897vf5Uc/+hF/+ctfKCws5D/+4z9G4lSCIAhC\nCLu9M1R+pdzauijrsWxVruUJx7JXrZrDrl2VuFzTsNkOsnr1EtaufWvQNYTYYkTE+sEHHxx0+2OP\nPTYSywuCIAiDMFSCmc1mHdSVXVn5Nk7nKsBCb6/G2rXrhxR2IbaIuQQzQRAEYXjoojxYTHewTmWD\nubyHEnYhthCxFgRBmIAYu5vpncrsdk1c3uMUEWtBEIQYYai+3SfSz3swK/rpp+cgLu/xiYi1IAhC\njDCYNVxVtXTI7UdjsOQzcXmPX0SsBUEQYoShyqhOpLxKEscmFiLWgiAIMcJQpVjm7S7a2t6ltJSw\nS3ywphojYUXLOM3YQcRaEAQhRhjKGjZub2t7F6dzFU6nconX1T1AaelU7r770mEL6XBF+ETc78Lo\nIGItCIIQIwxlDRu3l5aC0xlxibvdZ/KnPy06rjahwxVh6W4WO4xYu1FBEARh9IluGQpqPvXxCOlw\nRXg4IziFzwaxrAVBEMYRuku8ttaPx5MCLAQ0CgoODXuNoWLjQ51LktTGHhFrQRCEcYTuEr/hhv+i\npiYBeBY4xNtvu3G53ANiz4PFp4crwlLqFTuIWAuCIIxDmpsLgF5gOWChtVWjomJg7Hmo+LSI8PhC\nxFoQBGEcEG0hFxT4qK+fwrFiz5IkNjGQBDNBEITPiI4ON+Xlmygt3UZ5+TO4XO5hf69byPX1V1Fd\nfT0QoLBwN8dKAJMksYmBWNaCIAifEdEu6V27KqmtvS4cZz5aSVW0hdzcXEBt7SIqKgaOyDQiSWIT\nAxFrQRCEz4howXU6P09FRe2Qgmx0WRcUNFNf/xSQAXgoKPAcdUSmznCTxKRbWWwjbnBBEIRBOJbL\n+kQwu6RdwLts3Up4/aO7rBOBa4DFwLWhzyNHtJu9oqJ2RNcXTg6xrAVBEAZhNFptOhwl7NpVidP5\neeBdYCW9vRaqq9X6DkcJfX3r2LEjDjiMz5cWLsdqbs7B7AbPOalriUYS0WIbsawFQRAG4XjFaziW\nuM1mpbb2OsrK3KSkFA5Y32azkpychNv9bdzun1JTsyJs4UZb3QUFLeHzLVv21Elb/pKIFtuIZS0I\ngjAIw+3ypROxxDupr3+RurqXKS6OHxD71WPI5eXPhCxq8/pDvSREJ4r5fAkmy/94eoMb0WPV+/Yl\nUFhYSXZ2ETNn9kgiWowhYi0IgjAIx5tFHRHZGmABbvcWqqvT2LXrCWprrx+QrOVwlODzPcL27V1A\nNj5ffzhuPdhLQnSiWGnpNoZr+R8teczo7geNuXNlslYsImItCIIwCMfbajMisunAFvTOYk7n4kE7\ni9lsVpKSUnG7vwdYqKnRSEpaf9SXBKPotrXtAcoYjuV/PCVhEquOTUSsBUEQBmEoa3So7brI1tW1\n4HafyXAEcDChPNpLgtkKLqawsJK8vLMpKurl7ruHtvyPJsjH6+4XxgYRa0EQhEEYyhodarsusi6X\nm8svfwKnczFDCaAu+Pv3t6CSuoYnlGbRtZGXdzZbt15Bbm4GH3xwgPLyTYO6uo8myNI0ZXwgYi0I\nQswylo06IsLoBmrC9dD79iUQbaV2dLi55ZaXQiVXh5gzJ5kvfnEdzc052O0eVq26wCSkPl8PNTXf\nBzqBDVitXoqLE44plEcT3aO5uo8myDJZa3wgYi0IQswyGrXOwyXSMcwJ3Bauhy4srCTaGl65spYt\nW24Mb9u2bQNlZQG2br0CgPLyTab7sFofCO1rBa5l+vRnqaq6Ilz+NdTLiS66+/bF09HRRGNjEeXl\nz/Doo2VHdXWLII9/RKwFQYhZxjb5Se8Y9rzpGrKzi5g7V1mpBQUt+HwJvPZaErABWIgS4AyamvrD\nK0XfB2SjOphtAdJoa9uDyzXnmC8nuuhef/3TNDSswum0sHu3xo03PoHdjsSeJzAi1oIgxCxjmfwU\n6RjWhdGSnjmzJyygRotZ7fMgUAD00NbWTmkphnGWkTXmzQvyzjsP43SuwpgxPtyXE+Vuj+xXV6ex\nY8cVSOx54iJiLQhCzDKWyU+RF4WFDBVXHmgxfw5YRHLy7TidP8XptFFfr7Fgwe8oKzPex1dYtuwt\nnM7IsVu3gs023HKsQxhfIOCQuLonOCLWgiDELCMtQEdLWIv+bvXqOUReFAI4HFcOSG6LtvyhO/T7\n01Eu7nSgiwMHMnn11SVHPba3N5He3psoLKwkK6uIjo697Ntnp7z8mQGx63nz0qmp2YCawNXF/Pky\nHWuiI2ItCMIpw2Ax4fvuu5yVK2upqwvgdicDl1FfP5nBktmGEvTaWj8eTwrKCteAg4BqdgIaHR2V\nA65F9xps3Qq9vX7Uf8dv0NOTwFlnfUJDw3SczgwaGjz4fM/z+OPfCl8DJGG1eoGDzJuXwaOPXkN/\n/4BTCBMIEWtBEE4ZBosJR7fbhI3ANYPGi4dKALvhhv+ipkYD/gDk4PMlocqyAGpwuQoHWMjmHuEp\nqGQ2C273Il5//U7g1vA1bd/+AKCEuqRkfTjWDarrWVaWdch51sLEQMRaEIRThsES1gbGndOJjhfr\nFvXWrRj27WTz5oMUFf03gcCh0PbbAQuapqGywy3ActMYzGhr3eEooa7uZdzuyDX090+PuqZsQL0s\nqPGa0h70VEPEWhCEU4bBEtYqKl41CbjV+j7FxS5TIlnEot5AJLHrRYLBVSGR1YDHMQusDzWF+OjC\narNZ+fKX+9myJXINOTkHaWszZ4+D7hno5ni6ngkTAxFrQRBOGWw2azhG3dSUSUXFq1GJZB4cjuUD\nEsn27YtHucctwL1YLFPQNLMQQzvmDG0L8Klp2/vvv0lJyRFmzQqYXOIWSwD1IqASxs49N430dHP2\nOOiegSWha0mjsLABh+O6UXteQuwgYi0IQkwQnby1atUcKivfHvFWoyfSFa2jowmIxImTk9fg9Z6F\nWZyt6CIKO1BW9W3AfcDZwBG83ttoaNhCQ8P1pvM2NxcAV4XPd/jws2zYcMWA61Cegc2hZ+LG4bgO\nTYNlyzawd2/KZ96SVfjsELEWBCEmiBbRXbsqw4lUx9NqdLDyrNzcjPD3J9IVbfLkqTidG9FLsU47\nrZDZsz1s3/4A3d2ZBAKTQmumAW8CPwXeAGzAOcBiw2rpA8473OYvg5WyRbcy/SxbsgqfHSLWgiDE\nBNEi6nJN40QSqQaznJ999vrw98cSxsHEvrPzU4yW9ZEjlfzqV9excmUtjY2pHD78AR5PBt3de1Ad\nzGxEOp8ZO6C5gJ1AO3v2OLnhBicPPbT4pJq/yDzqUwMRa0EQYoJoEbXZDtLbG/mcn38oPOSioKAZ\nSAxNtTK7fo8lXqtWzWHnzntoa8sjPv4Q3d3puFzu8PGDiX12dpGp21h2dtGAkq/k5DXARcAelCir\nzmeZmR34fHfg9Z4P7AXuBiz4/VqosckLJCWlnrC7X+ZRnxqIWAuCMGocz4jLaOty9eolrF0b+ezz\n+amuVpOt1DSsaxisucn+/QHgSeBrwOQB4lVZ+TYtLbOAawgGLWzbplFREXEdDyb2M2d2snu3uT94\n9H59fRcBS1Au7/tISSmktBQcjjKWLXuL+vqrgM2mYyCD7ds/xe3+HsNxYw/2PB2OEpKTN4Zi1tIT\nfKIiYi0IwqhxPMlcg8Vjq6rs4d+Xlm4jInQZGEVv61bYtesJnM6bUC5oNYayuHjKAPFSmd1O1DSt\nLmAhdXUBGhubqKx8m/37W4gujVq1ag67dlXick3DZjvA6tVl3HnndswJZkfC1wNTuOyyI1RVqa5j\nEeu3K+qYLlQN9fDc2EM9z6efvkaaokxwRKwFQRg1oq3PfftSB8xr1jSGZX2b3b0ejKKn+mqvRu8+\nBhamTz+DqqqBGdXRmd2wAbd7Epde+if8/n9HdR4zD+6oqKgNJ7v19mosXVpJd7debuUDmoHvh86g\nAclApP+ncQ71oUP30NMzlbi4w8yblw4khLqfHduNLfHpUxcRa0EQRo3oeGpHx14aGswZ3sCg1uJQ\nfbhVL20f8ERo3URgAZFsbDia6EXHn5XYXoXfHwh9tqLizVU0NZ1BRcWrNDamYRTJSBexxSjX9lVA\nDSrT+wPgX2lufi18zqMNJHG53CQlDS+5TOLTpy4i1oIgjBrRceh9++wGoeykrq6Vvr4pKAt1IWAN\nW4tDuXxVL20Vu1ax6eXo4lVY2EBeXvCoojdz5pFQ/LkTeDG09QXgI4zdydzun1Bfr85dWLiWgS5v\njYgrezLqheFFIAd4gYKCwYV0sLjzcEutxnJkqDC2iFgLgjBqRFuU5eXP0NBgFkTzAI3lYWtxKJev\nUbCUIK4LZYV7cDiuO2YmtX58bW0LHs9PDedfh8redtHb68bvj8S0s7KmM3euOmdb27s4nStQrvh7\ngQySklbS359Mf/9cVDvQBcBfBj3/8cTxT0bYhYmFiLUgCJ8ZRqHdv99rGl6RkuKntHR92FocyuUb\n/QJgFLSKilePWfqkH19auo36eqM73A/00tX1KZr2C4wx7Vmz+sOu+VtvPURPzyaOHPkYv//HgA2f\nL5Kdrr94NDfnDCq2xxN3PpFua8LERMRaEITPDKPQKnd2RIxLSzEJ0VAu32gB7Orq5NVXf4guaD7f\nOh5/fNmAc+vH7dsXT0dHE93d8Zhd25OBa9G05zCKqdXqxeG4ElDiWVNzI2ZvwDVEZ6dDGna7e1Cx\ntdu1YcedJaFM0BGxFgRhTDhW/HWopKxoAUxMXItR0LZvjxtwzOHDxjnQG1HZ4CrrOzPTS3d3C8Hg\nTaG9zVOtiosThmy4EkloM2enT5q0i9Wrl/G9731EtNg+/XT04BBJKBOOjYi1IAhjwtEypI9GtGD2\n9+dgtpAPDzjmpptqQhncnahJWJF4dFzcM+Tnazidk0N7LwDuwGqdQXFxAqtWXRAuN2tr2wMUo2q5\nXUyatAuLxU1m5l407S7a2s5HDez4MZdc8is0bSrwGCpbXDVoOZ77loQyQUfEWhCEMWG43c2i9yso\n8Jmszby8Vlpa9PGSrfT2urDbN2GzHWDTpjJmzLDz8cdqAIfK1r4NYzwaDvPHP17OkiVr6OubgcXy\nMZdcks4f/nAlmgaXXfY4LS0/ALYA55KYuJaUlCx6erLwes8EvkZv72Ss1gdQHcwUfv+Foc9DN2g5\nFif6QiNMPESsBUEYE4abPBW934IFv+OKKx6hrs5CMHiY/v4errjiEIcPp/L++014vSo5rLfXRXHx\nL5k9+4vs21cPfA7owezG9jFvXjq//e1H9PWpnt2appGVtR5Ng5KS9bS0fAEl1KpEzO/vxu83J5Op\nuHU2Q3U0G6pBiyAMFxFrQRDGhOEmT0Xv19xcQFvbuwQCqrlKe7vGe+9VUl9/RSimq++7Ba/3Lhoa\nLMDVKFFNxyioU6Y0AqezdStE13qvXFkbcp13o4+1VEQnk6k1580LkpS0nrq6AG53K8aOZhJrFk6W\nURfrkpIS0tPTiYuLIyEhgT//+c+jfUpBEMaI4xncEZ08ZZyqZTx2sCSrDz4wj89U4zTBZjsQmtTV\nCfQxUFQvJTPzfk4/fSYdHXvp7k4JZXfrDVKeBRIpKPDQ1FRApGb6d6huZQNbnVqt71Nc7MLh+Ao2\nmxWXy80ttzzP9u1/ALKZNy+Iw/GVkXvIwinJqIu1xWJh/fr1TJ48+dg7C4IwrhnKtR0t4qtWzaG7\n20Ni4lr6+3PIzm7i7bcTaWubA3RTX78E2ExV1VJWrDiDmprb8fnsQCtvvHGE9HSLaXympn1ISclL\n+P3dJCSsIRBIAaZjdkt3A5NJTw8wa1ZPqO3p86HvazDXSa8LvSQsAZ4DJmOxrCEjYzpz5/aQlGRs\nxLLc9EJis1l5/PFvfRaPWziFGHWx1jSNYDA42qcRBCEGGMq1HS3iu3ZV4nTmoeK8GbS3dwA/wxgH\nbmrKZN++JhYufJ5gMNKk5PDhDcTH/4NJk9agaTPw+/fh9f6UhgYbEXd3MlBCxPX9D1QG94N0d/tp\nbEwNradPwUrH7GrPCZVYbWbfvgQ6OtxkZ5/HzJlHcDiWhsW5o8NNRcXwPAmCcDJ8Jpb1jTfeiMVi\nYdmyZXzzm98c7VMKgjBGDFUXHC3iym3dBhgbjAxsKnL11c8RDH4u6rsM+vtn0t9fTmFhJU7nl1FC\nrH/vB3YDS1HWsga8AawGLHg8GocP672+F6Ji1fuARabr1jOxy8s30dCwCqfTEuopHvEWqNptFdeu\nr19CX99fSE5OEvEWRpxRF+uNGzeSm5tLR0cH3/72t5k5cyYXXnjhaJ9WEIQxYKi64GgRV7HlwtBn\nN7AntIKKERcWNrBq1RIuvvgg8CEDZ0C3As/jdPaihHmx4ftEYAZKhDOIWM+6ld1FZmYuUGmYnnUd\ncB9wNoWFDTgc14Xv6WjeAn1spr7+jh1xuN3SHlQYeUZdrHNzcwHIysriyiuvZPfu3UcV69zcjNG+\npDFF7m98M5Hv70Tv7fBhNzfdVMPHH6czY0YXjz66hKwsszX56KNlrFixMbRPN2vXfotLLnmMlhYN\nFS+OuMCnTbuPd965iRUraggGVwGfALejYtDtKCs6H7gUaADsqIEaBSj394LQmkYSME7n6u6+j6lT\nz8XpXBzeIzW1kEWLjvDwwzeZrr+oqMf0olFU1EtubgZOp41ob4DF8qlpm9Np+8z+zkzkv5sw8e/v\nWIyqWPf29hIMBklLS6Onp4c33niDm2+++ajHtLd3jeYljSm5uRlyf+OYiXx/J3Nv5eXPhePRu3Zp\n9PUNZk3G8+tfLzJtOf/8PGpqNgD6HGkACzk5Z9LfH8/evSmh7XagAvgD8AVU/PkHRIu8Emz98/6o\n78wtSW222WRkfAw8hbK+D5OW9iF7987lO9+pNrmvf/zjL/DGG5W4XNOw2Q5w221ltLd3UVjYgdHi\nLyxs4ItftFJTY9zm+kz+zkzkv5twatzfsRhVsT506BA333wzFouF/v5+Fi9ezCWXXDKapxQEYYQY\nbhnWiQ6baG4uQLXhfAyj6L3zzm7OO28PZ51lrImeDEwFFpGfX09Ly2Sik8JU0xPlyk5IsBIIGL/L\nMp1j5swedu7sBH4Y3tbevoH29qvCCXB5eWdjt3fi8/nD7u7eXo21a9dTVWUfxOWvXOdJSdIeVBh5\nRlWsTzvtNKqrq0fzFIIgjBLD7TA2VFLZYOValZVvG9qGHgkd14MxvqxpBTidNxIM3kNZ2XoaG1Np\nb3+Pnh6NuLg/cs45mZx//jq2b+/A7Y4khcFeVCMSK3APA/uFb8Bq9VJcnIDDcTnnnVdLdOKa/nun\n8/M4nUuor9ewWv/IYC8jQ7UClRi1MBpIBzNBEAZluBbzUEllt9zyElu2qGzv+nqNF164g0DgrvDn\nBQvWsWDBOmpqfIbzgJpkZaGz024Yp9kTfnHYts1Fbu6DdHUB3IPFkk1c3Kf09/8EJdQagUAPkYSy\nbiwWK0uWBHA4rgx7B1SSmwvVSjQNleR2KcqKj7QKhUMYhV+6kQljwcBZcoIgCCiLWYkUgMb+/R9S\nXv4MLpfbtJ/NZuW++y7Hbvewb18ql1/+BCUlz7FtWwuqMxiAhUDAjlH8X3stgaSkROLjm4GvEmnr\n+S7gQtP2hs9lfnF4hvb2NPr7LwJmoWnX0N9fBNRgtT5KYWElKhltOSpLfDmTJ3cDsGzZW+F72LSp\njEmTfhnabwnwMzIz/5NJk+5AJao9BbiYNy+DsrL1nHfes5SVrT8h13ZHh5vy8k2Ulm4b9BkKwrEQ\ny1oQhEFxOEro61vHK68ECAS6cLu7qK6eyvbtf+Svf/22KX5tdJmDhtO5EZXBvQG4FiX6jRgt1N7e\nZKqrl2Ox3INxUIYS2Pvwem+jokJ1MTO72l1EN1BRMenFTJv2JJ98YkH913Ynykp2091dSHV1PHBZ\nKCb9MHl5ZzNp0gy83sgLRFzcJLzefwuvXVhYyUMPXXfStdLDDSkIwlCIWAuCMCg2m5Xk5CQCAWPj\nko20ta3hllseISkpNRx/bmxMY2AfbjXVCjaj6qKTgcdR86SnAN9ATbmaiu76jhxfAPyOLVuslJc/\nw+rVc9Bd7Q0NGVHJY16U21qjo6MJj2cFSvwvBHYBd4X214UdnE7V5ASexBzbzjZdR17e2SPS1ORE\nk/AEQUfEWhCEIYkWGV2Et2/vwu3+HrqlOGXKHShhzkANuuhFiV8Lqq92I5oWaRmqLG5QrmY/8L+Y\nG5skAT+jr+9RqqsTqav7G8XF8Tz99Bxuuul5tm0zCmwLaWk9/PM/r6exsQin02ilw8Dr7yISz+4h\nM/NeZs48C7vdg8/Xbyq9Gqn49FBJeIIwXESsBeEURs/YdjptFBZ2DCjPihYZFVceaIEePpyAcRBG\nQsJdBAIbUNnZk5k82YXbbRRNN/BrlKtcubaTk9fg988kGExBNTbRXd7fwe22UF2t3MdJSQA/B+ag\nLOrvEx9fFWoN+gy7dycSEWM9acyGPiHL6+3E6707fK3p6ZVs3apmTbtc7lEpvRoqCU8QhouItSCc\nwkTHmqNjqQ5HCUeOPMJrr2kEAk7i4rK45JKH2Lv3CGoaVTdwMYFAPkbxPuusc5g5s4emptcGtVhV\n1na84RgbfX3TgY+BuahxlQuAHAa6jzNRLvUl4evs6ckIX++WLY/Q16eL8SLgDmy2WcyfH4fDsZxv\nfGMnu3dH1szOLgqvM1Q51skyWusKpw4i1oJwCjBUg5NjxVJtNitPPfUvpm3l5ZtoabkFXXgtltvR\ntHOIxH5d7NnzNh99VITNtodHHilD0+CVV+7E75+NillfS2Lievx+o4A3Ar8wrZuRkYHHE+0+1qiv\nbzadr7+8jtPXAAAgAElEQVT/AKWl27DbO5k58wzee89oxV/A7NkJVFVdBsDMmUdCAzkiDVIEIdYR\nsRaEcc5wOo0NlY18tFjqcAVe07JQFnEVqnd3F8FgJb29quPX0qWVzJ07Db//34kI8wbmz8/kf/7n\nDrzeuSh39udN606ePJudO6/kllseYfv2LiAbn6+fn/98Hps3dxAMRlzdmvYL6uvVveXn/wJz0th7\nfPRRIeXlz+BwlIhLWhiXiFgLwjgnWojr6h6guDjPJNpDWdC6cKmYtcskXMMVeNU0pNLw+bemc7lc\nBQPOHxfnYc8eNxbLLJQrfSHK9R1Zd9IkJwBJSanhZLaaGo2kpPV85Svp1NQsN5wzsnZPTz6RjmgN\nwApcLls45l1VtXTYLunhtlwVhNFGxFoQxjnRQuh2n0l19SKM8eehLGg9ljrYoITIum6ghq1bCZdR\n9fWtY8eOOI4c2Y/ff6bp/Kq1Z+RcmtaI3T47dP5O4EWCwSAtLbOAr6FqoTcCC4iLu51g8MvAEVpa\nfkBFxeZBXzSefnoO77xTidM5DeVWj2SS9/a2ohLXQL0IbEHPAt+3L/64BFjqo4VYQcRaEMY5g2ds\nm+PPJ+L6LShopr7+KVRJViK9vUuorp7MSy+t4Z/+yYbb/R3gEeAAZrezF2OrT6/XRm1tVyi2nQXc\nZth3I3ANKSl9XHbZ0/zP/2Tg8fhQXczcvPiik/nzC03rt7Q0cMstzbhc01D/hX0/tE4asAO/f7ph\n//0YG6h89NEdfPnLTtzunzAcAZb6aCFWELEWhHGOLsR1dQHc7kkol7Kyns1WpMbTT885DjduIsZy\nLF1Yvd6LeP31N1EW60qUtfwEaiBHO6qLsdFFvQGP59rQPtEzoPuA5wgEGnj11UmGLO6rgY34/d+n\noeEOpky5k9bWTCCHlpZp1NT0oCzq7xPp7f0ukAt8E3gUVfaVazqf13s+Xm8iwxVgqY8WYgURa0EY\n5+iubJfLTUVFbbhcyuG4nIqKod24RiEvKurh7rsvNQl5c7O5bEpZyhpwhP5+O5GuY1ZUE5PridRG\n3wecg5o9nQc0AR+iyq6Mk7KSgCX4/Xpf8IENWDyeWSQnt2O2yO8AZgPVqBeEbuAW8vP/MzQ+MxX4\nDip2bbT6+1CW//AEWJLRhFhBxFoQJgjRtbwdHW7q6gIMZUVGx2O3blWJafooy/37WzAL3QcoUfwq\nmnY/0EYkVmxsF2pDCfWi0P7LUeJ9F8oK3xD6eRi4OXSMGo9pPp9qwKJp+4AZmIXc+HKgERdXyeLF\nm1m9+uusXbuerVuht9eC8jJsJDXVj9V6EKdzRegY87jM4T5TQRgrRKwFYYKycmUtbncyRgFsa3sX\nl2vOoCVYbvdsqqt7eeGFZwkEfoBqevI4CQkHiY930dcHyjLdiKaBGtDxBMpSNSd5KYu6m0gnsjwi\nVvi1obUnh36BalGqhFUJ//+GjrkTTZvOpEmfmu7DYslG0yLXnpmZHxbVqio75eXPhLK/rcByFi3a\nyN13XxdOWLPbzeMyBSHWEbEWhAmKEuPLiCR7fYDTuWKISVYaKuY7hUAgDlV+dROwhUDgCwQC/wvM\nAv7VsP8TKAs3A9iHxXIP8fF5BAKTQts04K+AhylTPqa11Xiut4F+LJZ7yMgoJDHxQw4f/hg4HdUi\nVG9peit9fRZaWlwUFlaSl3c2druH7m6LqT/4vHlB071Hu68ffngJ/f3xYiUL4xYRa0GYYOix6P37\nA8ALRMqjGgAL+/bFU16+iX37EigsrKS7Ow+PJxXoR8V6VwHPM3Bs5YOYXdFeIq7opSQn34HFkkIg\ncD1qulYkOe3ccx8hGFxDe7sVSEFlmF+IpnnxeBaQmPhbIuVWoCzvFoyu9by8s009vCsqjLHkr5ie\nQbT7OitrYGmaIIwnRKwFYYKgi3RdXWu4NElZqA8CU1GZ0y/S0dFEQ8Oq8PcLFqwjI8PCn/6Ui7KI\nLaj4cXTCVzbmmHIbcC9wJtCL16u3En0KJeSRY1991UJcXAoqSWwjymqPZJn7/YVRax9BxbUHTwST\nWLJwqiFiLQgThEjC2POYRfZzKMsYrFYv2dlFoVnO6vvm5hxefPEqtmypxOPJRAnkQuBhzHHoD4B7\nUOVQn6Cs9dNQ/43obvR7UWIcB/wB1VAlm2CwjWBwNsYs78j1pQE+EhPvxO+/ECXUXwX+jB7DLixs\nwOG4bljPYbCmJ7m5GcN/kIIQg4hYC8I4xihMjY3NKGs0Oqv6g9C2i0lNbaGpqdXwvYuWlgYuuiie\n1FQfHs8hIsM0koG1wNmo+dR+4N8M696DuQ57PxExVo1U4EbD9/eGfkZf35vArcyf/0fee68Bl6uA\nYPBBEhPzSEhwMW9eBg89dJ0pGexoXcgG6zr27LPXj+RjF4TPHBFrQRjHDBxxuQFlFW/AYulE0yaj\nksImAz/B6ZyDsnZ19/X7tLTcTkuLGiepYtjxeDyRrl9qNvVUVDmWbhF3hn4+jxLfhahY9FMoV/jn\nQvsaLegzQ9fXgYpPzwQ+4owzTufsszfj82XidN4aPm9f30ZgOe+8U3nU+46uH5euY8JEJG6sL0AQ\nhBMnWpiURfssYCEpCVSZlJVIzPkaVLz4Z6i4snnSVV7e2cTFTQltawLuIxA4DTW+8gPUCwGooRv/\nhnKTXwO8SE5OV+j35aiMbo9hfw34O6rL2b+EzvuvQCVnn51OVdXSIZqwdOJ0JvGlL71MefkzuFzu\nQe/bKMh2ux7rVueVrmPCREAsa0EYx+Tnt2N2KX+KsliXk529FqfT+F02A8XQQ3QS1/79h4hY6SsN\nx98R+mVHZY4b1+rC69Vbieq11A+jeocfRjVKuZWMjN/j9f4Wv/8H4WN1oR28x/mLwG243Zbw1Kyf\n/ewC3n//TZStoWq5jYIsXceEiYiItSCMA4aK0VosAZRL+xxUYtZNJCb+ioUL13PTTZdTVnYHXu8Z\nKBHXk8f0lqC7gHxgLSkpUykuDuDz+QkGe1FCrTcyIfTzDOA6VH11E+aXhAz6+oyNS14GvoDKLs9A\nxbxtBAIF5OYewOmcjHLHv0hjo5fzzvt/ZGbmUlhYSUdHPl7vx8BZKE+B2YK++urn8HrvDp970qQ7\ncDi+G35WkikuTERErAVhHDBUjLa5uQBlyR5BWco1zJ49i6qqpZSXb8LrvQtdnJOSKklIuJ2enjhg\nGiqurGqws7PvIzm5kOrqG9HHWMI+zILsDH13AGVdr0G9JAAsJCnpMIHAGjStCJVsdhvKotbLxzR6\nexPp7b2JwsJKenoScbt/gsdjwePRcDo3AuUUFq7F6bwRlQneHzpeXdP+/V48Hv2zcu9bLGdIJzJh\nwiNiLQjjgH374ol0IusKfdZdx06MYyA7OytDx6QSsUq34PPdh893H2bX9qNAKocPF1BX10JEBK8F\nqlCZ4fkogc5CDc643XD8htC+GoFAK5p2t+E7NaVLfc5An1kNVvLyzgagvn7g4I7U1Azi4n5PMHgm\nyoJ/mMREF37/atzugee12Q6OwBMWhNhGxFoQxgEdHU2ozmJKrDo6lCA7HCVs2/Ys3d0RIXc6M7nh\nho20tzehRk0aB20UYnZtu4Dv0NtrobdXL69agcoeTw/9Mo67/H3U8T5SUp6gtBS2bp0V9V1a6Pca\neXlO2toy0NuPFhR4SEpKHSRGrXHwYDvB4C8M2+8jIeE0/P7I2gkJXSQmPoHNdpBNm5aMwBMWhNhG\nxFoQxgHRjUy6u/MpLd2G3d5JWlob3d03YxS3mpofkJFRScQa34PK3Nbjyrqr20qk3MsKTCUh4X7i\n4vz4fBehksOMAtwedbwPTfuE1auXs2tXdUjw9VjyLs48Mxjq5Z3Ntm3Gmux14USwxsZUDh/eS1aW\nnVmz1g8i+oXYbAdMa3/taykSlxZOKUSsBWEcMHPmEXbvjoiVxzOJ+vqrqK/XyMy8n4Edyyx0dWWh\nOoFtQcWoVwE5KDd2GrAas8t6OZBIIHAPSsD/GZXR/RyRCVr5qASzT9AbpHi9LoqLf8mMGbPp6FiD\nxTILm62ZTZuWMWOGHYDS0m2ma2xuzglN7oL4+ATmzp2KwzEfm83Keef9P5Mwx8W9z2OPLeI3v5EM\nb+HURcRaEGIUYwZ4QcERFixYR3NzDvv3f4jbXR7ay0JcXA4DO5Y9h7Ki70fVNH+CyuZuAaaj/ukb\nBb6XSExZjzHXYIyFq45lFtRkrFzD8Vvweu/ivffUfmVl66mq+qHpXqLLsvLzD1FSsh6n8/NAN/X1\nSwA1DWzTpjKKi+/A650LHCEY/Cm/+c1msaSFUxoRa0GIUaIzwBcseCRUB52NcZqWGg+5jr/+tZfu\n7k+Bi1CW8OmoxiMbMVvRG1ATuIwCvxeoNHzuIjLUg9DPPOC7od8/aTg+zbSfPtXLWGbmcJTQ17eO\nHTvigMP8/e+dtLYas8U3huutZ8ywc+aZc0ICrpAuZMKpjoi1IMQI0bXU+/aZrd/t27twu7+HLqiZ\nmfeTnh7gwAE7s2YFuPRSqKkxCq4+0jJ6cEYGkEpy8hr6+opQJVnXAveRklLIxRd3smePO9yCNLJe\nu2Gdr6EsbTvwIcaBH8apXvX1Gjt33oPXO4nu7kwCgRRUh7PJRJLZrEAaBQVt4WcRbYlLFzLhVEfE\nWhDGEKNAt7Xtwem8CbBRX69RWFiJ2fo1dyCLi8vB6VyK07mFhgYbCQnNmEVZH2kZPTiji4yMOI4c\nyUEN2zgHZWmfTmlpAJhMS8vNqCSyDajGJMmobmcuVAw8DeU670MN67gPOJ1Jk97D5TIniLW0nAbc\nYDi/XtJ1DsrVvhyVABeplT6RLmRHG+4hCOMdEWtBGEPMgzjK0OueIZ3u7ngWLPgdzc0F2O0efL5+\namoiohsMtqISwFTcNxBIxizKHwLrAB8JCXeQmmqnt7cVvz+Prq6bQsdGyrL0TmDLlr2FuW3oY6hE\ntXZUDFwvq1qMst5/HfrcH2rCsiHqOj7GPPAjncjMaj9KvFfQ3Pxa+LmcSBeyW255iS1b1JSv+noN\nn28djz++7LjWEIRYRcRaEMaQgYM4VN0zWPB4FvHOO5U888ylVFa+zYEDqRQWVpKdXcTMmT3s2NGD\nx6N3KFPlUCrTuwg4hEokawb6uPLKqTz++DJKS7dRX38VqtXnFNO5LZaZVFS8SkGBL6r+OQnV4/t7\nqKYo0Znnt6EEWo9xL0QJsB/1wvBjIrHpDSi3ezfqBaAGZWWfvKtbxcONYQOZUyRMHESsBWEM0F22\n+/cHUMlaKlksISGDQCAiOE7n5/n615/D6VyFXtvc3d3K4cOddHbOwFwjnQccxOxyvg/4N/7+919Q\nWrqNtrY9QDHKlW22xHt7J1Fd/VVycx8gLq6SYPBclKguBJ4hMfGXTJ6sceiQUcg7iMTBdXe7FWWx\nbwQuQAm1up+MjB4uuSSN5uYUCgr+Avhpbn52hMqx9AEk+rUdPsn1BCF2ELEWhDEgeg611foAxcVT\n8PniTK5u2EN7+ySU8H0K3IbHsxGP5ybMMWA97mu2llUi1ye0tEyipUUD4oiLewzoIRj8FuamKWnA\nb2lvn40S/UuIWMQp+P134fGsJGJFd6GsZz0uvjD0nQc129ofWidyPxkZbTz00HWjEkueNy+dmprI\ntc2blz7i5xCEsULEWhDGgGj39/TpZ1BVdQUul5va2kiNMXyf/v77gVtRcd9OlGgbY8CHgZ8Dp6Hi\nwy4iItsM/BfG0q1g8FGUqNeiXNyXoBLMsgFzJzRlraeg11/7fJ9HxbHdKBd2H6rZyixUL3Er8+f3\n8NFHHQZvQCRJzelcQUXF6NRMP/TQYpKSamlq6sduD+BwLBrxcwjCWCFiLQhjwGClSR0dbm699QW8\n3lTgXVQf799hsVhD+3Whz3c210w7iSR96f29P48S4FuBNxgqLq72vxPlEg9gdqufTWLiLvx+Y1xc\nb1eqZ3FvBCJWfn7+PVRV/V+WLXsr1B5VT1J7At3CHq2aaRmNKUxkRKwF4SQYrFxI0zhmCdGqVXPY\ntasSl2saNtsBVq8uY+XKWmpqMlGCGBHI/v5VKKH7J1Ss2Si8XSir1rgtK7R/AcrCPow5lpsVtX8i\ng7ce3cP8+Vbee68y1GnsCPA14uJuJxiczWA13IcO5bFs2VuG2Lhu4SeG1tyA3R44mUcuCKckItaC\ncBJEdxnbseNOLJZEWlq+SHQbTSOVlW+H3MRq2tXatetDFmc8oAshqCztIpYsWU9dXStudwCz8LpQ\n7m/jtkmoGHRC6LNuMetx5vej9j8Ns3gfAdYQF5cNTGLTpq+wdu3bNDVl8v77/43X+wsi5VnmGu5A\noIv6+u8BZRQWqpeR3t5EdDe61erF4bhyJB69IJxSiFgLwkkQHXtubc3E7KbeyL598dxww5Ns394F\nZDNvXj8HD9pMx+lWeH19AsqtHRHA5OSPqaqqoKTkJdzuuURiyZ+gBnTEoVzZXyAu7m0SE0/Dau0h\nP9/PO++sQpVyAVyKcksfQrnN1QuFwije7cDdBIMWtm1TLxL6y4YqrzKWZx1Cud3PRDVJ0T0IFvLy\nzmbu3E6qqyO13MXFCdKoRBBOABFrQTgJomPPaqqV0UpN49Chf9DQMBNVp2yhpkajsHAtRoFsa3uX\nRx5ZwvPPr6e/H2ANMAP4kOeeUz2yOzo+QM2n/hmq3KsIVaOs1igsrKS2dkVYDE8/3YHRnR5xbx9C\nJY3prURdpKTcyec+d0FoSMh0ol8kdDIzP6S39ymUlR4EWiksTCMvz0Jb236czhWhPbVQOdbxdyIT\nBGEgItaCcBI4HCXs2mWM6YJRhAsLG+juzid6KEZW1nSCwXtCrTgP4XTm8POf/5WMjM/hdn8ntJ+b\nxMTf8KMfHaCx8QX6+rJRTU+CqCEdlqg1i/jRj14KNQc5hNc7jYHu7TtQGdr5JCSsITGxCJvtIK+/\nfiOZmVmUl3dSXR003YOxWcm5506ltdU8lzovL4etW6/A5ZpDRcVmkzBL0pcgjAwi1oJwEthsVmpr\nr6OiQh9l6QbWceCAlY6OvWRl2Wlvfx9lyUYEsKOjiba2eIwNTF555U5SUuyoEqgkQMPvn857730F\n+CbKMs4Grg8d86RpzY8+eoeGBqMlvQqze/sQytJ+ArievLxK6uuVkObmZtDe3oXDUUJX1yb++te1\n9PfnkJfXyurVXw/fb0tLtOcgKyzmgwmz9OsWhJFBxFo45TlZQRlMpMrLN9HQsCpUvuQCfoXqo51D\ncvJenM6fAq9hFD6//zz8/q8DT2F0b0cGX6SjMrvNk6+s1qmkprbgdJ6FWUhPJ9L0pBs1IUvPFrdw\n6JCV8877T7KzizjrLB93330pNpuVjAwrfv8PUUM4VMz6vvsms3JlLR988AnGF4BJk/6Ow/HdIZ9N\ndAIerBdLWxBOABFr4ZTnZAVlMLGPTjxLTEwmISEPm+0AaWmz+fBDGwOzsveimo34MIuuPviiG5X8\npR8zGYsFtmy5iNLSF4nUQOvrdaJGUBpFXwM+ADz4fB04nbfjdFrYvVtj69YHKC7OGzCas6kp0/CM\nzE1OZs8+86gvNtHPQeZSC8KJIWItnPKcrKAMJvYFBUeor9cTsRrw+1fj96syrbi421GiOR01ZcuF\nSkzTUIMyEjGKrsXyDzTtDSAXaMNYhlVSMpnKyrfxeH6KEtInAC8WSxslJalYLI/wt78l0Nv7CX7/\n5NCx/4pqQ/p703273WdSXb1owGhOu91jeEZ6k5PNwCJmzVp/1Gcjc6kFYWQQsRZOeU5WUAYT+4IC\nH2ZXduT7YLAIZeUeRDUNKUSJbyJKjL9NxH39Ppr2A5S4bgD+D/AUcXFTyM9vZu3aMr73vY9QQl2D\ncnHXk5CQQnp6DqtWzaGy8m2ami6goaGVQOBaw5V7MFvi3YCFtjYbmZn3Ehc3hXnzgjgcX6Gi4lXT\nM7Ja36e42HXM7G7JBheEkUHEWjjlOVlBGUzsm5qMiVjdmEVxHzAXZSk3oTK0dSt6DZo2GX1spGpu\nYkW5x53AfwM/Ixi04HSqeLLdrlFf/yKq8cgW4Iv4/X+jujqVHTueprVVTzp7LOo6JqFqtnWL/VpU\nYxM3Hk8u8G2SktZjs1kHeUbLhxXXl2xwQRgZRKyFU56TFZTBxN5siS5g0iR9OEcD5vnOD2O0ujVt\nKuaksNND3+k9wfVhHjVAOi+++AlPPTWX6upGlFDrDUgWAxtob3cb1r8KPclNZZtnYh7c8SAwFfg+\najZ2JCQgoisIY8uoi/Xrr7/O2rVr0TSNq6++mu9+d+jMUUGINYzJY0VFPeGM6ejv7HaNp5+eg6ZB\nRUUtH3zQR1LSSgKByWhaFgkJCeTkvMGhQzMwzneO7tsdH3+Q/v7lKOFNA/4GrEe5rI3DPJSL3e9f\nxHXX3YHqIKYnkaWH9rMQF5dNMOgCnkPVWbtRZWT7UF3QjIlsn0OJPOgxdIkxC0JsMKpiHQwGufvu\nu3nsscfIy8vjG9/4BldccQWzZs0azdMKwogRnTzW1xfJFDd/52LXrofp6cnH7U5GtQA9D11Uu7s1\nurvvZaBLvBdju87MzG5crl+i3OTdKGv6d8TFHSYYfCp0nC7cABb6+magyrgexNyx7A4CgWyUq/si\nlBv9bsP390ZdS1doTY3MzGYuv3y9xJgFIUYYVbH+xz/+gd1uZ+rUqQB87WtfY9u2bSLWwrjhaJni\n5u+2hAdzRFzKUzBbrlNR85/10qckoAKYTGbm/Vx+eT61tfmodqLGcqtzCAZ3EElYM2drJyU10tc3\nGbgg6nznh873o9Dn56K+n0JcXCWZmflcfHEQTfPT3PxsyJX/LWleIggxxKiKdWtrKwUFBeHPU6ZM\nYffu3aN5SkE4LvQZ0sYhGw899NWwUB0tU9z8XRpmIcxhYLb1p6h48JbQfsbM7CyqqpYye/bTUesk\nomZbzyYya/paVO/wmUAj55+vMWXKeurqWnC7jefrwzzCMtqqn0QwuBq3WyM9fSO//vWiE3+QgiCM\nKqMq1pqmjebygnDSRGZIR4ZsJCVFXN3G5LGiol7uvjviFjZ+19a2B6fzUpT1qgGfEB9/iJSU/Xi9\nOaSmdjB3bhJJSX/hwAErDQ31GIWzp6cJgNTUZjweo6C+jZqQFT2M42z07O3333+A555bSmNjE5dd\npiey7UG9GNQYzrMAWE1CwukEgx0Egz8I3YmFl1/uw+VyizUtCDHKqIp1fn4+Tqcz/Lm1tZW8vLyj\nHpObmzGalzTmyP3FFk6nMdlL/Xz5Zbj55s08/PBCiopO49lnrx/02I8//pitWz/C651OUpKb5OT7\n6euLCGt//4P097t5//2vMmuWPXzcsmUbaGiwY8z61rQscnMzyM+fTUuLMRu8ELOl3YNyg98U3max\n5JKbm8HNN+8OCfUSYD5KqA9jjImXlc3i2Wf/lWXLnuJPf5ocWkPD5UpizZo3ePrpa07mccY04+3v\n5vEi9zexGVWxPvfcc/nkk0/49NNPyc3N5YUXXuCXv/zlUY9pb+866vfjGX1YwkTls7y/kRoQUVjY\ngbI8jVauxp/+dA11dXdywQWn09ycg93eyaOPltHfHx8+trj4v/F6VUJXX9/AMiz4HL29izjnnDWc\nddaF4evcuzcF1RAl0go0Le1+PvjgAG1tjcBqIpb0PZhd1wdRLvGI0H75ywHa27tC6+qubiuwHKv1\nAdzuVeFrfuGFRzj33Cc57bROMjPvx+M5K3TMQvbufW3C/v2Uf3vjm1Ph/o7FqIp1fHw8a9as4Tvf\n+Q6apvGNb3xDksuEESE6S9vne4SkpNTjFm+Ho4QdO35Pa2ukhSf4AQutrZnU1NwYPseKFea4rsrC\nNoqzLvzmjmB9fRdRX78k3IpUNTHRO5KloHqEZ1BS8gRO57+gLO40EhPfRNM6CQSM1xYAesOJYfPm\nBXnooa8Aegx9Sfj4KVPexGJJQLnmu4EFBAIZNDRYaGhYQWHhWjyeReHrLShoobx8k0zIEoQYZNTr\nrOfPn8/8+fNH+zTCKUZ0lvb27V243SrufDzDOGw2K7m5X6S19RuGrZtRYmseB/nxx+mmYxMT38Pn\n08up9gNWLJZVaNoUVCb4wtA6R8JrNDVl8vTTc/D5nmf79k85csSH378aj8cSilXrE7bgnHOCfPCB\nJ6pF6BPAdSxePPD+VAxdnyftxuc7Pfyyoa7j58A09KSzrKzpzJ0bicd3dSXIhCxBiFGkg5kwLonO\n0lZzno8+jGMo13lHxweYLeJ/oCxRv2n71KkdpvXmzTuNurprUAKryq1UUuUToWP+GlrrJlQzkhfZ\nv99LRcWr3HnnpVRWvs3WreD361neVlRWOeiZ521tB+jtjVxDYuJHLFwYqX8+WjigtHQbZsv/QmAR\nqu5aY9as/rAY5+ZmcP75zx7zGQqCMDaIWAvjkugWnz5fPzU1Rx/GMdQozKys6TidxqQuG5BGUtLf\n8fkiLmhN8wMRgfzb3zKJuLKNomhDJXlp5OfXc/75f2H7dhdu909wuy1UV2vs2lUZVZetZ3nvITPz\nAOnpnTQ2FnHWWekEg/fQ2WnHZjvIpk3fZMaMSLLa0cZ75ucbx2lG3PLJyZlkZ1fS2FhEefkzOBwl\n5OZmyIQsQYhhRKyFcUl0r2qXy01SkhLv/PxD+Hx+Skqeo6OjiezsImbOPGKY0+wGati6FcrLn+G0\n047Q0BA993k+gUAzxlpop3MzYBbIwTuBvQtYsFrfp67u/2KzWSkt3UZ9fUTQW1ryMQu8L3TeFfT2\n/gaPZzVOp1qvrGxod/TRmrZYLAHMDViUWz47243TuSo8xxrW8+yz18uELEGIYUSshQmBUbzLyzdR\nXX0jSvwiohSZ01wDLKe3V1m5Cxaso6xsPXV1AdzuScDngQcJBmcCT6JaeU5mxoxuYKBAKsv7DlQ8\nuAOYDnhITvawbNlb2O2dFBT4TFZrMNiIWeCdwCpAw++fynDd0UezhpubC1DDO9TLSUrKc5SWQmNj\nUagP95kAAB2VSURBVOhFwLy+DOsQhNhFxFqYcETE1Ni9y0J2dhFz565n61bo7Y1s37LFD8SRlLSP\nSy9NYceO9/H7jT221wCn8cYbbXz8cdMQ8fJ/Ae4HvoxyN/8Tra1NtLbGU1+vYbM1oAR9BtCIGk/5\nKOACckhI8HHmmU9y8KATtzsXo5C3tb2Ly6WGhETHp49mDUeuU5VxlZYqC728/JmQRS3ubkEYL4hY\nCxOOiEh1YU4QcwNJJCe3mJK21Pzoa+nr0/jb39bQ3z8Ts+V8EbAEp1Nj6dJKamuvo69vHVu39hMM\ndqFqnp/D3GnsTuDfw59drmYiPb9dwC9DP28DLAQCGtOmraOjw4/bnYkS9rMAC07nCioqlAteud87\nqa9/kc2bXyQ//xCbNpWZ4tg6Qwm5uLsFYfwhYi1MOHQx2rcvno6OylDMugefzx9yj3cCG7Bavbjd\nzcC3ULHddPr6JqHKsIyWc6T0yuWahs1mJTk5iWBQj1u7gD9hFvjZUZ+Nru0tqOlYz5v22bEjLtTA\nxAIsxVjGFXGFW1Bu/GsIBi3hF4j6+h8OeA5DubXF3S0I44+4sb4AQTgROjrclJdvorR0G+Xlz+By\nucPf6WL05z/PZ+7cacTHJwAaBw7o7nErcC3Tp2eRnNwN/A8qE/tS1HCM6cDtKOt3FZAMPAW4sNkO\nAkZXuxvVuSwdJewQGdox1Gd96EdX1D6HMQu8uYzLbu8M7Wd277tc007gCQqCMJ4Qy1oYM06mZejR\nSpaG2ieSYBaJ1WZm5vL66z6MFmvEoq4M/VKfU1LuZNOmbwJGV3sNKiFtPpFe3/XAdeidxPLz/8E5\n56Tw1lsPEAza6OvbT1/fYlR2trLwi4sT8PnSTOVn8CbgJjHxQ1avXoamESr5ysKY+Ka/QAiCMHER\nsRbGjOEI7lAcrWRpqH2ys4v4whfWsWNHHHAYny8Nl+t0Iv20zRYrmMurZs/+AmvXvk1T00cUFPhY\nsOB3vPZaGr293ai49TWhdeopK3s93EnM4bjB9BLicrmpqNBjxgEcjiux2azh8jOVld4K3ArY8Ps1\n1q5dD2CqzY6LqyQ/HzZtWjKsZyYIwvhFxFoYM4YjuDC4BR6dkd3W9i6NjbOprHw7FKtuort7CkYL\n9PDhvRw4kIjb/RNAjcMsLFwL5KFi1p+iOnzplu1HGC3xDz98h927fwxsob5+Cvn573DxxbBt23J0\nK1qNpnRTVXXLkPetu+n1+9LLuxyOEqqqluJyufnSl17G7Y5MBDPHrNXPL3zhbLZuveL4HrogCOMS\nEWthzBhux6zBLHCHoyTkEv48cASncwVf//rDIctT1Vfr61qtD5Ca6sfpXAG8gVHwsrKm09PTh9t9\nLSr+vBHoBeJJT7fS3R3pbOb1TkElhy0HOmlp6aalxQU4UAlk76EakMwIdwY7mls/+r5eeukOzjjj\ni8yceYR587yDdGTTpMOYIJyiiFgLY8ZwSog6OtzU1bWiMqe7gIXU1QUAyMs7G6cz4gJWiVadKAs5\nsj9kk5WVHJpdbSznctHR0YT6Z2BM9Ipj0qQP+dKXckNWs3EQhje0dgORUiy9i9mtqBj2tVRXR9z6\nQ8Xmoz0LXu9cdu9ewu7dkUYtA5+NlFwJwqmIiLUwZgynhGjlytqw21qJ4gbc7klUVNSGRk1GLE2b\n7QC9vS+i1y4b909N3R/6HEnqSk1tCVnincCDoTOqY71eDYvlEcrK9CYqicBpgHGKlTG+fQ7K6s4I\nb9Nd10PF5gc2V4mUiG3fHsfOnZcPsMyl5EoQTk2kdEuIaQa29vQBC2lqysThKKGsbD3nnfcsZWXr\n2bSpDKvVO+j+2dlFoX1fo6wswM6dV5KXdzaRUq5CoMh07JtvJlFVtZTSUg3l+p5i+F5PSoOI0Kah\nLHe1TXUecw8am+/ocOPz9ZCYeCeqocq9wFfDx+ovJIIgCCCWtRCj6K7j/ftbMDcoSQYmY7d7BrXM\ni4vfCrmgzfvPnNkzYF+zZbsA1S50cfjYI0eacbnchvi4RiQBbQEqLn4xSqi/Sn7+b9C0Plpbn0OP\no1dUbB7gAbDbPaxcWUtNzfdRVv2LZGZm0Nv7K/z+81Gu9oU0Nb02sg9VEIRxi4i1EJNEXMeq21hm\nppf09BaysuzMmrWeVasuoLx804A4sB4Hb2xM5fDhvUPuv2LFGezc+Qlxcb9H09rQtGyU5RwZien3\n9/GlL71McXE8mzYtYfHi/6Kt7V5UMtmnXHppOllZ7tCam3E4bmDZsrdobdXj6CrePm1aIYWFkU5q\nDsflLFv2FsYGLTNnPovdnkF19VVE9wQfbu25IAgTFxFrIWYwJmIpi7oTo5ht3fp/wvuqyVqROPCO\nHXdywQX/v717D66yvvM4/s4dSAI5QIBEuiGAEay2TC11YVxCsY0SwKBopXWkRZuV0sEx7Qw3124t\n3VBTrbZDhyJip1AqWNYkUAhVA4RWKcvWTTEqZYg0CLmS5DQJhlzI2T8eTs41yUlyDufJyef1jyR5\n8jy/x4if/G7f379QVTWelBQb+/bdicVyj5frjbra+/f/DZttKvZtXUbFskSMFd23AGeBWVitVyks\nXAgcYO7cmRQUrMAepnFxO/rorR/qPsMabMye7VhwVlv7IcYsVAuw8PqCMc8V7mvXHtA8tYgorMU8\nPM+Jfg3jPGnPbUru88A1NaMpKjIWf5WW2igpeZ709AleVl4bVcpsNvszXgVGYdTyjsGo8x2O8yEc\nsIeKitFERUW4PLOqarzHO2zYcAenTm2msXEyHR2tdHZ67iNft+6oS3GT5OTN5OU9isWS4LHCvbfj\nMUVk+NACMzEN9wBOSLjavXjMfZuSo0421/853uV7rdYZFBau6F6k1VNdbSOclwMP4Dhw4zxGr95+\nTSy1tR9y7twZl2c6/wJhr1V+773/Q2VlCq2t99HZGef1evf3nDDh1u6hbvf30l5qEQH1rMVE3Lcy\n/eu/dhET00RFxWjWrj3iUmTEfcjY4LywrAXn3qx9LrukpBqr1blKWTze64I7evUjRpyisvJx4G3g\nBSIj4/nqVyPIy3MMs3uOCuwBFpGQ8DyTJ6fS0HCW8vIUsrPfICmpvcfiJjq+UkS8UViLaTiOthxF\nQ8NZ3n13NE1NI4H5lJaOwbl2uMWSwNGjj/LUUwc5caKZrq6RjBr1X3z6adL178nEOQjtK8dd63I3\n0dLSRXGxZ4979OirhIe/CtTT1TWRq1dPYN9j3dlp429/20xj4z9Zu9Y+x96Ja489DhhDevpE4FPK\nyjZQWRlGWZmNhQt/1UPBEx1fKSLeKazFNOxBlZ2dT1mZY07Xfq6z+/ytxZJAdPQorNYngDCamowg\njI6OoqLimNeeqc3m8hG5uf9Gbu4ujh6toqnJ0eP+9NOP6ezcdP3j3TiOtQQIo7LyNpYuzae6ehoQ\nAdTg3LNPSDhDenqj28pv43urqpJU01tE+kVhLUHR2/GYnoVQjLlfb/O37td6C0LnZ9XWfkBl5SPA\nCUpLLZw6Vcirr36ZoqIyjCpmxtx3Z2eM030XERb2PDabYw82NFJdzfW2NQNfJyrqP/nsZ79w/ZeE\n5S7z0KrpLSKDobCWoOjteEz3cHPupdo5iqZ0Ar/AmKOezJkzZzl/fjqpqSlenwVZwHPAOowe8hKW\nLv0B7e3P4dqTHwf8DmNOu4nY2Gu0tPwEo6zoFYzKaP/h8j2xsVO89pg1Dy0ig6WwlqDo7XhMz3Bb\n7lIYpKHByoIFu64vLmsB/oF9q9XVqzbuv38zpaVrenyWUVrU8XFbW6rb12MxVoR/B8ee6k20tKzC\nqP8dS2Rkk8u2LIhlzhz7QjdXmocWkcFSWEvA+XIetfPQcF/h5r5PGTbjHLaNjZNdnlldfRqoAyYB\nTURHv097u+PZMTEfc/Wq4+Pw8L8QE3Mzra2OeyYm3sq8eYc5e3YkKSlW2tvDXY6wTE4u46WXHvXr\nvyNVLhMRO4W1BFxP51E79557Kh/qjWtP+Z/ANYzDMIxqYBbLRS9D369h1P22MW9eM7Gxjmd/97uZ\nfOtbRiETi+Ui+fnfIDfXtcb41KmfsnfvCurqjIM6GhutREc79/4fHVS49jYtICKisJaA8zbk7d57\ndi8f2ltYTZpUh2Pl9SGc545HjPgB+fkP88QT53Ad2o4HrEAR77wziowMG3v3Oupul5be7vKMvDxj\nq1hP88z+HtrubVpAREQVzCTgfKnK5R5Wb74J2dlv0Nho7b7GXiXs3XerMHrKBzAWejm+b8aMO0hN\nTfFS4awZo/DJclpbV7hUN3O/f0ZG8fUiLF9mz547AFi27CSf+cxmFizY79Euf1DlMhHpjXrWEnB9\nrYb2drBFa2sUhYXLgV0899yXWbfuKCUlnVitMcDNGNXGwFix7Riurq39kIwMSEq6wsKFO6iqGk9S\n0mWgg2PHYl3mod17r84nfZWWHqKk5C1Gjap2mR+/eHEPZWUr8PcwtVaMi0hvFNYyIN4WRCUmxvf6\n9Z7mdD0XjD0HrMIeqJ6lPH+CUdP7MAAjRjzDzTfPor7+LJWV36Gy0kJpqY2srF28+ebd3W2Jiemk\ntXU39pO2ej4cxCg9arWGYbXux3PPt/+HqbViXER6o7CWAfG2IMo4PrLnr/cURp5bq27FOBrTGA72\n/PoM4JcYx1oa27UmT95BRMStVFZauq9zPuXKOewTEp4nPX2i18NBjLY6lx5twbPmuIapReTGUljL\ngHhbEFVfbyU7e7+X86h774m6b+OaNOk0V69eBuppb48lKanNrUjKOVpaEl32OZ84EU56uvftYO5t\nnTLlZrZv77l4iethHwtJTt7MuHFpNDaeIyHhM0yb5jgFTFuuRORGUFjLgHjbJ716dZHP51E7y8tb\nQFvbDv7yl3CgHputDat1JRBGUZG3gy+Wc+edr2G1Ovd468nLM+a43ed9fS332dNhH/ZtWYmJD3Zv\n3bLTlisRuREU1jIg3vZJL1z4v7ifRz1lSkGfC6YslgRiYqKxWpdgzEN3YAR9JpDgtd73nDlxFBW9\nhrElq5k5c+KwWBJYv/4LLFu2n7//PYk//nEbqak3M2WKY7GZL4u3+jN/rC1XInIjKKxlQLztk25s\njMJ5fjc9PdLrcLOd8xCyMWy+H1iBo7e8B1jutSf80ktLiI4+SkXFNVJSOsnLWwzAsmX7XRarffTR\nHj76aEX3YrPBcB7m96USm4iIvyisxS+MHuV8jIAdQVTU/1FefgvZ2W/0OI9rDCHbe9MzgCqce6kj\nR3aQkbGrx+pm3nq/jY2TXe7hz9XbzsP8PVVi05YrEQkEhbX4hdHDHIOx//l3dHQ8S1lZGGVlrvO4\nnr3p/wYex3FutKOXmpFB9/nWvs4LWyyf0NoamNXb5887rxL3XolNRCQQFNbiF3l5CwgL28mxY9do\nauqgq8v7PK7nnukXcD43Oioql8jIz2CxXGTjxvsAKC8fhXNIfvzxKI/n238JGDNmOg0Nz2CzJREW\nVk1q6nTS0nb5pcebmtrMqVMa8haRG09hLT5raLDy1FN/vL5q+zJz5sTx0ktLsFgSsFgSiI6Oxmpd\njrE4zHuouS/IioyMp7PTfu0YOjpS6ej4Bq2tNnJzd7F9ewoNDX93uV99/VngHpe2uf4S8DWysnax\nffsK/Gnr1kza2jTkLSI3nsJafLZu3VEOH7YPWdsoKnqN6Oij3cPAjmHiTGDP9TlnXELNfUHWqFEN\nxMUZ+5g/+eQ8Vms29gM39u/v5NSpXxAbm4QxFx4HtDB2bIpH227EquyxYzXkLSLBobAWn3lWEoun\nouJa99cdw8QJwHIyMjznlh2FRzqxWkfQ1PQdmprGMHv2LqZOnUBh4Rjsq8BttjAqK22MGPEMsAl7\nwE+btsujbVqVLSKhTGEtPjMC0V6TOxb4gKQkxypvX4aJ7QuyMjKKKS1d2v35iorR7N17B7CLwsJm\nHD3pZq5ds7gVRfG8r1Zli0goU1iLz/LyFnDy5C+prjZqcsMSYEf31/szTOzeE66t/ZCHH4aUFBsx\nMVW0ta3u/lpExA/Yvv3fe72fVmWLSChTWIvPLJYEJk26jepqx1B4VdX4Ad3LuSdcW/uhy2lZ8fHb\naWtzPCM19TZ/NF9EZMhSWIvPvJ07PdC5YeeecEYGLqdlRURYcV79nZbWNui2i4gMZQpr8Zn7udPJ\nyZvJy3t00Pd1HxKfMyee6GjNP4uI2CmsxWfuq8EnTLgViyWhuyBJZaWF5OSGfh8T6bk4bLGOmRQR\ncaKwHqYGcg5zT9ujPKuS9e+YyIEsDtM50iIynCish6mBnMPc0/aoG3VMpHNA19Z+QGXlasCic6RF\nJOQprIcJ957oxx/H0t+A7akH3FOP29+9X9cefBbGXuyv+9x+EZGhSmE9TLj3pJOTc+mpfndf3EN4\n40ajmIkxZ93Y3eMeSO+9N54V1GKv/1kVy0QktAUsrLds2cLrr7/OuHHjAMjJyWHevHmBepz0wT3o\nxo6dwuzZA1tx3VMIJybGU1fX3OMzB9v7de/BJyeXMWFCl1aMi0jIC2jPeuXKlaxcuTKQjxAfuQfd\ntGnXBtzL9TWE+1uvu69hc88580e1qExEhoWAhrXNZgvk7aUf/Fk729cQ7u8z+xo2V0lRERmuAhrW\nu3fvprCwkNtuu43169cTHx8fyMdJL/wZdL6GcH+feaNWlYuIDDVhtkF0f1euXMnly5c9Pp+Tk8Os\nWbOwWCyEhYXx4osvUldXR25u7qAaKwNXX29l9eoizp+PIzW1ma1bMxk71lxDyA8//Dtef91Y3Q02\nvva1Pezd+/VgN0tEJOgGFda+unTpEqtWreLAgQN9Xuu8QCnUuC/AupGys/NdCpdkZfl/X/Jg36+x\n0cratUddeuxmmpMO5s8v0EL53UDvN9QNh/frS8CGwevq6khMTATgrbfeIi0tLVCPEh84hpitQBFv\nvgnZ2W+YqvKX5qRFRLwLWFj/9Kc/5aOPPiI8PJybbrqJH/3oR4F6lPjAsSisCFhOa2sYhYUD3/vs\nbeW2L78diohI/wUsrPPy8gJ1axmADRvu4NSpzVRVTcJmG/wiLm8rtwsKVviruSIi4kQVzIaJzZvf\nu3685Wv0VrnM3mMuL4+goaGCcePSmDr1isdwuVZui4jcOArrYcIRrpnAHkaO7CAjA49tV44e8x5g\nA5WVYbz/vudwube91vX1VrKz9+skLBERP1NYDxOOcE0AlpOR4Qhf5/nnf/yjEyOA43DuOZeXjyI7\nO9+jHrh95faGDV9g1qxfcfHiOvpbC1zHXYqI9E5hPUz0VsjE9TSr3RjD5M04D5dfvnyGsrKnsQdx\ne/sOfvObh7vvkZ2dz8WLtzKQoXF/H/ghIhJqFNbDRG/bolznnxeRkPA8kycn09Cw+fqc9accPZqA\ncxCfOBHu5R4tDOQkL81/i4j0TmEtbvPPY0hPn8j27fe5XJOWthXnIIZ6L/e4D2OuO5bk5DLy8h4d\nwPN13KWIiDuFtfhU63vOnDiKil4D4oFm5syJ87hHTMxhzp4dSUqKtV8nYvnzkBERkVB0Q8qN9keo\nl5Qbqu/nSynQofx+vgjl9wvldwO931A3HN6vL+pZB0ioVfhSKVARkeBRWAeIKnyJiIi/hPd9iQyE\nVjiLiIi/KKwDJCXlnxirpkErnEVEZDA0DB4gWuGsymQiIv6isA6Q/i7ICsVgU2UyERH/UFibRCgG\nm+btRUT8Q3PWJhGKwaZ5exER/1DP2iRCseSm5u1FRPxDYW0SoRhsKqQiIuIfCmuTULCJiEhPNGct\nIiJicgprERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgpr\nERERk1NYi4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NY\ni4iImJzCWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicgprERERk1NYi4iImJzC\nWkRExOQU1iIiIiansBYRETE5hbWIiIjJKaxFRERMTmEtIiJicoMK68OHD7N48WJmzpzJBx984PK1\nbdu2kZGRwcKFC/nzn/88qEaKiIgMZ4MK67S0NLZs2cLs2bNdPl9eXk5RURGHDh1i+/btPPvss9hs\ntkE1VEREZLgaVFhPnTqVKVOmeARxcXExmZmZREZGMnnyZFJSUjh9+vSgGioiIjJcBWTOuqamhqSk\npO6PJ06cSE1NTSAeJSIiEvIi+7pg5cqVXL582ePzOTk5LFiwwOv3eBvyDgsLG0DzREREpM+w/vWv\nf93vm06aNImqqqruj6urq5kwYYJP35uYGN/v5w0ler+hLZTfL5TfDfR+Q12ov19f/DYM7tybXrBg\nAYcOHaK9vZ1PPvmECxcu8LnPfc5fjxIRERlWwmyDWKb99ttvs2nTJhobGxk9ejQzZszglVdeAYyt\nW/v27SMyMpKnn36au+66y2+NFhERGU4GFdYiIiISeKpgJiIiYnIKaxEREZNTWIuIiJicacN6x44d\nzJgxA6vVGuym+NXPf/5z7rvvPpYuXcrjjz9OXV1dsJvkV3l5eSxcuJCsrCzWrFlDS0tLsJvkN73V\nwh/Kjh8/zr333ss999zDyy+/HOzm+NXGjRuZO3cuS5YsCXZTAqK6upoVK1aQmZnJkiVL2LlzZ7Cb\n5Dft7e089NBDLF26lCVLlrBly5ZgNykgurq6uP/++1m1alWv15kyrKurq3n33XdJTk4OdlP87tvf\n/jb79++noKCA+fPnh9x/gHfddRcHDx6ksLCQlJQUtm3bFuwm+U1PtfCHsq6uLjZt2sSOHTv4wx/+\nwMGDBykvLw92s/zmgQceYMeOHcFuRsBERESwYcMGDh06xJ49e9i9e3fI/Pyio6PZuXMnBQUFFBQU\ncPz48ZAsW71z506mTZvW53WmDOvc3FzWrl0b7GYERGxsbPefW1tbCQ835Y9gwObOndv9TrNmzaK6\nujrILfKfnmrhD2WnT58mJSWFm266iaioKBYtWkRxcXGwm+U3X/ziFxk9enSwmxEwiYmJzJw5EzD+\n3zJt2jRqa2uD3Cr/GTlyJGD0sjs7O4PcGv+rrq6mpKSEhx56qM9r+6xgdqMdOXKEpKQkbrnllmA3\nJWBefPFFCgsLiY+PD6lhK3f79u1j0aJFwW6G9MJbHf/3338/iC2Sgbp48SJnzpwJqQJUXV1dPPDA\nA1y4cIFHHnkkpN4NHB3T5ubmPq8NSlj3VG/8qaeeYtu2bbz66qvdnxuKvZi+6qnn5OSQk5PDyy+/\nzG9/+1vWrFkThFYOnC/14rdu3UpUVNSQmyscSC38oWwo/v0ST1euXOHJJ59k48aNLqN3Q114eDgF\nBQW0tLSwevVqzp07x/Tp04PdLL84duwY48ePZ+bMmZw8ebLP64MS1j3VGz979iyXLl0iKysLm81G\nTU0Ny5Yt4/e//z3jxo27wa0cOF/rqS9evJgnnnhiyIV1X++Xn59PSUnJkBw1GEgt/KFs0qRJVFZW\ndn9cU1Pjcx1/MYfOzk6efPJJsrKy+MpXvhLs5gREXFwcX/rSl/jTn/4UMmH93nvvceTIEUpKSmhr\na+PKlSusXbuWvLw8r9ebasI0LS2Nd955h+LiYo4cOcLEiRPJz88fUkHdl4qKiu4/FxcXM3Xq1CC2\nxv+OHz/OK6+8wtatW4mOjg52cwImVHqkt99+OxcuXODSpUu0t7dz8OBB7r777mA3y69C5WfVk40b\nNzJ9+nS++c1vBrspftXQ0NA9PHz16lVOnDgRUv+//N73vsexY8coLi7mZz/7GXfeeWePQQ0mnLN2\nFhYWFnJ/0V544QXOnz9PeHg4ycnJPPvss8Fukl/9+Mc/pqOjg8ceewyAz3/+8/zwhz8MbqP8xLkW\n/qpVq1xq4Q9VERERPPPMMzz22GPYbDYefPBBn1amDhXf//73OXnyJFarlfnz57NmzRqWLVsW7Gb5\nzV//+lcOHDhAWloaS5cuJSwsjJycHObNmxfspg1aXV0d69evp6uri66uLjIzM0lPTw92s4JGtcFF\nRERMzlTD4CIiIuJJYS0iImJyCmsRERGTU1iLiIiYnMJaRETE5BTWIiIiJqewFhERMTmFtYiIiMn9\nPyQ+uNKCpR6MAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0xa813090\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "# Plot the Data (Optional)\n", + "\n", + "import matplotlib.pyplot as plt\n", + "\n", + "plt.scatter(inputs.numpy(), labels.numpy())\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JaFHyAG9nDET" + }, + "source": [ + "## Step 2: Define our TensorFlow variables\n", + "\n", + "We'll use Keras's object-oriented [`Dense`](https://www.tensorflow.org/api_docs/python/tf/contrib/keras/layers/Dense) layer to create our variables. In this case, we'll create a `Dense` layer with a single weight and bias.\n", + "\n", + "(**Note**: We're using the implementation of `Dense` found in `tf.layers.Dense` though the documentation link is for `tf.contrib.keras.layers.Dense`. When TensorFlow 1.4 is released, the documentation will also be in `tf.layers.Dense`) " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 34, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 22, + "status": "ok", + "timestamp": 1505502830753, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "z9r-ZeyrXu3A", + "outputId": "6230a7a3-29fe-4d08-f101-da80425bad82" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 4, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Create TensorFlow Variables using Keras's Dense layer.\n", + "\n", + "wb = tf.layers.Dense(units=1, use_bias=True)\n", + "\n", + "# We can access the underlying TensorFlow variables using wb.variables.\n", + "# However, the variables won't exist until the dimensions of the input\n", + "# tensors are known. Once the dimensions of the input tensors are known,\n", + "# Keras can create and initialize the variables. Until then, Keras will\n", + "# report the variables as an empty list: [].\n", + "\n", + "wb.variables" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "docKLUaonYG_" + }, + "source": [ + "## Step 3: Define our loss function\n", + "\n", + "Our loss function is the standard L2 loss (where we reduce the loss to its mean across its inputs)." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "0_w8ZJSCtuY7" + }, + "outputs": [], + "source": [ + "def loss_fn(inputs, labels, wb):\n", + " \"\"\"Calculates the mean L2 loss for our linear model.\"\"\"\n", + " predictions = wb(inputs)\n", + " return tf.reduce_mean(tf.square(predictions - labels))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 34, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 24, + "status": "ok", + "timestamp": 1505502830875, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "RkNbXoXkpjVH", + "outputId": "c36fc98d-3a57-4074-901d-c10ae017ae3f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "\u003ctf.Tensor: id=40, shape=(), dtype=float32, numpy=7.3549819\u003e" + ] + }, + "execution_count": 6, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Test loss function (optional).\n", + "\n", + "loss_fn(inputs, labels, wb)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 51, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 57, + "status": "ok", + "timestamp": 1505502830981, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "K_7beXoHOU7t", + "outputId": "1ad0856a-02ec-4117-a6c0-b41030981d87" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "w: tf.Tensor([[ 1.56891453]], shape=(1, 1), dtype=float32)\n", + "b: tf.Tensor([ 0.], shape=(1,), dtype=float32)\n" + ] + } + ], + "source": [ + "# At this point, the variables exist, and can now be queried:\n", + "\n", + "w, b = wb.variables\n", + "print(\"w: \" + str(w.read_value()))\n", + "print(\"b: \" + str(b.read_value()))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YIlebeb_qYtC" + }, + "source": [ + "## Step 4: Create our gradients function using `implicit_value_and_gradients()`\n", + "\n", + "With a loss function defined, we can calculate gradients and apply them to our variables to update them.\n", + "\n", + "To calculate the gradients, we wrap our loss function using the `implicit_value_and_gradients()` function.\n", + "\n", + "`implicit_value_and_gradients()` returns a function that accepts the same inputs as the function passed in, and returns a tuple consisting of:\n", + "\n", + "1. the value returned by the function passed in (in this case, the loss calculated by `calculate_linear_model_loss()`), and\n", + "1. a list of tuples consisting of:\n", + " 1. The value of the gradient (a `tf.Tensor`) with respect to a given variable\n", + " 1. The corresponding variable (`tf.Variable`)\n", + "\n", + "Test it out below to get a feel for what it does. Notice how the first value of the returned tuple (the loss) is the same as the value returned in the cell above that tests our loss function." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "v1spZQ4NwW1U" + }, + "outputs": [], + "source": [ + "# Produce our gradients function. See description above for details about\n", + "# the returned function's signature.\n", + "\n", + "value_and_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 153, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 46, + "status": "ok", + "timestamp": 1505502831114, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "21WMcpsmFFLd", + "outputId": "f51b3171-33f5-4f87-8bf7-0be2dc8edc8a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Outputs of value_and_gradients_fn:\n", + "Loss: tf.Tensor(7.35498, shape=(), dtype=float32)\n", + "\n", + "Gradient: tf.Tensor([[-3.00773573]], shape=(1, 1), dtype=float32)\n", + "Variable: \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e\n", + "\n", + "Gradient: tf.Tensor([-4.06519032], shape=(1,), dtype=float32)\n", + "Variable: \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e\n" + ] + } + ], + "source": [ + "# Show outputs of value_and_gradients_fn.\n", + "\n", + "print(\"Outputs of value_and_gradients_fn:\")\n", + "\n", + "value, grads_and_vars = value_and_gradients_fn(inputs, labels, wb)\n", + "\n", + "print('Loss: {}'.format(value))\n", + "for (grad, var) in grads_and_vars:\n", + " print(\"\")\n", + " print('Gradient: {}\\nVariable: {}'.format(grad, var))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "JVDWpL9VYWdP" + }, + "source": [ + "## Step 5: Create an optimizer\n", + "\n", + "We'll use a `GradientDescentOptimizer` to fit our model." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "DudNEebMKDWN" + }, + "outputs": [], + "source": [ + "optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.1)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "YBeJYxY8YaiO" + }, + "source": [ + "### Step 5a: Test Our Optimizer\n", + "\n", + "Now we have everything needed to start fitting our variables to the data!\n", + "\n", + "In the next cell, we'll demo these capabilities. We'll:\n", + "\n", + "1. Print the current values of `w` and `b`\n", + "1. Calculate the loss and gradients\n", + "1. Apply the gradients\n", + "1. Print out the new values of `w` and `b`\n", + "\n", + "You can run the cell multiple times. Each time, you should see the values of `w` and `b` get closer to their true values of 3 and 2." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 102, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 103, + "status": "ok", + "timestamp": 1505502831285, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "diDZfrMJM3OC", + "outputId": "d585fff0-ecb3-4e98-9b33-bbae07a95d8c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Values of w, b, BEFORE applying gradients:\n", + "(array([[ 1.56891453]], dtype=float32), array([ 0.], dtype=float32))\n", + "()\n", + "Values of w, b, AFTER applying gradients:\n", + "(array([[ 1.86968815]], dtype=float32), array([ 0.40651903], dtype=float32))\n" + ] + } + ], + "source": [ + "# Test the optimizer.\n", + "\n", + "print(\"Values of w, b, BEFORE applying gradients:\")\n", + "w, b = wb.variables\n", + "print(w.read_value().numpy(), b.read_value().numpy())\n", + "print()\n", + "\n", + "# Calculate the gradients:\n", + "empirical_loss, gradients_and_variables = value_and_gradients_fn(\n", + " inputs, labels, wb)\n", + "optimizer.apply_gradients(gradients_and_variables)\n", + "\n", + "print(\"Values of w, b, AFTER applying gradients:\")\n", + "print(w.read_value().numpy(), b.read_value().numpy())" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "61TgeLVlKEQp" + }, + "source": [ + "## Step 6: Create a training loop\n", + "\n", + "Of course, now we can simply turn all of this code into a self-standing training loop. We'll also capture our loss and approximations of `w` and `b` and plot them over time." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 397, + "output_extras": [ + { + "item_id": 1 + }, + { + "item_id": 2 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 225, + "status": "ok", + "timestamp": 1505502831550, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "VukGe-huNaJ4", + "outputId": "f0a8d665-1910-477c-d8ab-c94ccdc4afcd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[2.111051321029663, 2.3047544956207275, 2.4602210521698, 2.5850086212158203, 2.6851789951324463, 2.7655951976776123, 2.830157995223999, 2.8819968700408936, 2.9236228466033936, 2.9570505619049072]\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAFXCAYAAADnFpTQAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4FFUbBfAzu+m9koSShBQCSC+igIAgRRGkChJEiggo\nHURAEBQBQeADRcWCha50ULFLk6IivYRQQwskhPS6O/P9sckmm4Rkk2x2difn9zz7bLuZvC8JHO7M\n7FxBkiQJREREVOlUchdARERUVTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMjArdlJQU\njB8/Hk8//TS6d++OkydPVnZdREREiiMY8znd6dOno2XLlujbty80Gg0yMzPh4uJijvqIiIgUo9TQ\nTU1NRa9evfDbb7+ZqyYiIiJFKnX38s2bN+Hp6YkZM2agd+/emD17NjIzM81RGxERkaKUGroajQbn\nzp3DoEGDsH37djg4OOCzzz4zR21ERESKUmro+vv7w9/fHw0bNgQAdO3aFefOnSvxa3g5ZyIioqJs\nShvg4+ODgIAAXL16FbVr18aRI0cQGhpa4tcIgoC4uBSTFSkHX19Xq+8BUEYfSugBYB+WRAk9AMro\nQwk9ALo+jFFq6ALArFmzMHXqVGg0GtSqVQsLFy6sUHFERERVkVGhW7duXWzdurWyayEiIlI0XpGK\niIjITBi6REREZsLQJSIiMhOGLhERkZkwdImIiMyEoUtERCbRuXM7uUuweAxdIiIyCUEQ5C7B4hn1\nOV0iIqKy+OijFTh69BAEQYUhQ4ajU6fOuH8/HnPmzER6ehq0Wi2mTJmOJ59sgwUL3kZU1HkAArp3\n74nnn39B7vIrDUOXiEhh5s6dhd27d5h0mz169MLcue8aNXbv3t9x+XI01qz5Fg8eJODll4egadNm\n+PXXn9Cq1eN48cVhkCQJmZmZOH/+POLi7uGbbzYBANLSUk1at6Xh7mUiIjKp06dP4qmnugIAPD29\n0LRpc5w/fw716j2CH37Yha+++hyXLkXD0dERtWrVwp07t7F8+RIcPXoYTk7OMldfuTjTJSJSmLlz\n3zV6VloZCq80l/e8ceOm+Oijz3H48EEsWDAXAwcOxuDBA/D11xtx9Ohh7Ny5DX/88StmzHhLjrLN\ngjNdIiIyifxwbYbff/8VoijiwYMHOHXqBOrXfwSxsbHw8PDEs8/2wrPP9sLFixeQmJgIUdSiffsn\n8fLLoxEdHSVzF5WLM10iIjKJvLOX27d/EmfPnsbQoS9AEFR49dXx8PT0wp4932PjxrWwsbGBk5Mz\nZs16G7GxsXj99TcgSSIEQcDo0eNk7qJyCVIlrThv7esjKmmNR2vvQwk9AOzDkiihB0AZfSihB8D4\n9XS5e5mIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIismjHjx/DmTOn9M93\n7NiKn3/+0STbXrv2K5Nsx1gMXSIismjHjx/D6dP5odurV1907fqMSba9Zo15Q5dXpCIiogrbsGEN\n7O3t0bfvAHzwwVJcvnwJK1Z8gmPH/sGPP+7C7NnzDMZHRV3Ahx8ug0aTDWdnN7z55hx4eXlj8+ZN\n2LlzG2xsbBAcXBujR4/Fzp1boVbb4Ndf92DixNfx779/w8nJCQMHDsa4caNQp04ETp48gczMTMya\nNRdr136FK1cuo2PHzhg5cgwAYMaMqYiLu4fs7Cz07/8CevTohVWrViI7OwvDh0eidu0QzJ49D7/8\nsgebN2+CVqtB/foNMGXKdJOuE8zQJSJSGOe5s2Bv4qX9snr0QloJiyg0btwM3367Hn37DkBU1AXk\n5ORAq9Xi1KkTaNy4mcFYjUaD5csX4733liEsrBY2bdqGTz/9CDNmvIX167/Bli27YWNjg7S0VDg7\nu+C55/rqQxYA/v33b4Pt2dra4Ysv1mDz5k2YPn0KvvpqPVxcXDFgQC8MGBAJNzc3zJw5B66ursjK\nysLIkUPQvn1HjB49Ftu2bcaXX64HAFy/fg2///4LVq36Emq1GkuXLsIvv+wx2awaYOgSEZEJRETU\nRVTUeaSnp8PW1hYREXVx/vw5nDx5HJMmTTMYGxNzHVeuXMakSa9BrVYhO1sDHx9fAEBYWDjmzn0T\n7dp1wBNPdDDqe7dt2w4AEBoahpCQUHh6egEAqlevgXv37sLNzQ3ffbcBBw7sAwDcu3cPN2/GoH79\nBgYrIv3779+4eDEKI0cOgSRJyM7OhpeXV0X/aAwwdImIFCZt7rslzkorg42NDfz9A/Djj7vQsGFj\nhIWF4/jxf3H79i0EBQUXGi0hJCQUn3zyZZFrL7///gqcOPEfDh7cjzVrvsSaNd+W+r1tbe0A6BZc\nsLW11b8uCAK0Wi2OHz+G//77F5999jXs7OwwbtwoZGdnF7MlCd26dceoUa+V40/AODyRioiITKJx\n46bYuHEdmjRphkaNmmDHjq0ID69TZFxgYDAePEjEmTOnAeh2N1+9egUAcPduLJo2bY4xY8YhLS0N\nGRnpcHJyQlpaWrnrSktLhaurK+zs7HD9+jWcPXtG/56trS20Wi0AoHnzR7F37+948OABACA5ORmx\nsbHl/r7F4UyXiIhMonHjpli79is0aNAQ9vYOsLe3L3I8F9DNit99dxGWL38fy5cvQnZ2Dp5//gXU\nqhWId96ZnRuwEvr3HwhnZxe0adMOs2a9gb/+2o+JE183OLGppJOc8t5r1ao1duzYisGDn0dgYBAa\nNGioH9OzZ2+89NJARETUxezZ8/Dyy2MwefJrEEUJtra2mDx5Gvz9/U32Z8Sl/R5CSctNWXsfSugB\nYB+WRAk9AMroQwk9AFzaj4iIyOIwdImIiMyEoUtERGQmDF0iIiIzYegSERGZCUOXiIjITBi6RERk\ndt99txFZWVlyl2F2DF0iIjK7zZs3Iisrs9j3RFE0czXmw9AlIqIK27BhDbZu1V0n+YMPlmLCBN2S\neseO/YN582YbjN2yZRPi4+MwbtxovPTSSwCAzp3bYeXK5Rg2bBDOnDmF/v17Ijk5CQBw4cJ5jBs3\nCgCQmZmJhQvfwciRL2H48ME4eHC/uVo0CV4GkohIgbyaNyj29YRjZ4p9vazjCyvL0n79+g3Et99u\nxIcfforQ0BqIi0tBZmYGGjRoiLFjJ+aOMry8Y94lHb/5ZjWaN38UM2a8hdTUVIwcOQQtWz4Ke3sH\no+qUG0OXiIgqrCxL++lIuTcdtVqN9u07Fnq/qH/+OYpDhw5g48Y1AHSLJdy9G4vAwGCT9VKZGLpE\nRApk7Ay1vOMLK9vSfkXZ2dkbLF6gVqshirrgzc7OP+FKkiS8++5i1KoVWKF65cJjukREZBLGLu0H\nAE5OzgbL9RVeeycgoDqios4DAPbt+0P/+qOPPoYtWzbpn0dHR5myhUpn1Ey3Y8eOcHFxgUqlgo2N\nDbZs2VLZdRERkZUxdmk/AOjZsxemTh2PgAB/LFmyssgSfUOHjsR7770DFxcXNG3avMDrL+ODD5bi\npZcGAgD8/QOwaNH/Kq8pEzNqab9OnTph27ZtcHd3N2qjFy9ehKdnQIWLk5OSlpuy9j6U0APAPiyJ\nEnoAlNGHEnoATLy0nyRJZfrc1IABA5CTk2P0eCIioqrAqNAVBAEjRoxA37598d1335U6/sSJE/jw\nQ+uZ7hMREZmDUcd0N23aBF9fXyQkJGDYsGEICQlBixYtHjq+Ro0aWLp0Ebp164769R8xWbFERETW\nzKhjugWtXLkSzs7OGDZs2EPH/PDDD3j22WfRvHlzHDlyBDY2/GQSERFRqWmYkZEBURTh7OyM9PR0\nHDx4EGPHji3xa7p3747nn38B3323EXPnvosJE6aYrGBzUdLBfWvvQwk9AOzDkiihB0AZfSihB8D4\nE6lKDd34+HiMHTsWgiBAq9WiR48eaNu2bakbfvfd97Bv3594//2F6NatOyIi6hpVEBERkVKVeiJV\nrVq1sHPnTuzYsQO7d+/GK6+8YtSGPTw88f77y5GdnY0JE8ZAo9FUuFgiIrJMsbF3MGTIAJNuMzr6\nIg4f/kv//ODB/Vi//huTbFuupQUr9YpU3bo9g759n8d//x3DqlUfVea3IiIimRW+wEVFXbp0EUeO\n5Idu27btEBn5kkm2XdLSgpWp0s9wmj9/Efbv34tFi95F165PP/SSYEREZN00Gg3eeWc2Ll68gNq1\nQzFr1tuwt7c3GHPr1k0sW7YYSUmJcHBwwHvvLYCLiw/++OM3fP3151Cr1XB2dsHy5R/jiy9WITs7\nG6dPn8TgwcOQlZWJCxfOYdKkaViw4G3Y2dkjOjoKiYkPMGPGW9iz53ucPXsa9es3wMyZcwAAS5a8\nh6ioc8jKykKHDp0wfPgrBksLenh4YMWKT/D330fw5ZefIScnBzVq1MTMmXPg4GD6lYsqPXS9vLyx\nePH/MGxYJCZMeBW7d/8MtVpd2d+WiKjKmjvXHrt3m/af9x49NJg7t+TdsTEx1zFjxhw0aNAQCxe+\ng+3bN2PgwMEGYxYvXoBp02aiRo2aOHfuDObOnYslS1bim2++wLJlH8HHxwdpaamwsbHByy+PRlTU\neUyc+DoAYM+e7w1m06mpKfj0069w8OA+vPHGJKxa9RVq1w7BiBEv4tKlaISFhWPUqNfg6uoKURQx\nYcIYXLlyyWBpQTc3NyQlJWLNmi+xYsXHsLd3wPr132DTpnUYOvRlk/4ZAmZaZah79x7o1asPduzY\nhs8//wSjR5d89jMREVkfPz9/NGjQEADQtesz2LLlW4PQzcjIwJkzJzF79hsFFjjQ3Tds2Bjz589B\nx46d0b79k0Z9vzZtngAAhISEwcvLG7VrhwAAatcOQWzsbYSFheP333/Grl07oNVqkZBwH1evXkVI\nSBgKLi149uwZXLt2BWPGjIAkSdBoNGjQoFHF/0CKYbYP0C5YsAQHD+7HggXvoEuXbrlNExGRqc2d\nm1XqrLQyFD6mW/gQrySJcHV1w5dfrte/lveRoalTZ+D8+bM4dOggRox4EatXryv1+9nZ2QEAVCqV\n/nHec61Wizt3bmPTpvVYvXotnJ1dsGDB2wbLBObXJaFly8cwZ867ZWm3XMy2tJ+Pjw/ee28pMjMz\nMWHCa2W6ljMREVm+2Ng7OHtWty7vr7/+jEaNmhi87+TkjICA6vjzz9/0r124cAGA7lhvvXqPYMSI\nUfDw8MS9e3fh5ORksPxfSYq7zlNaWhocHR3h5OSMhIT7OHLkkEEtedt+5JGGOH36JG7dugkAyMrK\nxI0bMWXo3HhmvVRUz5690aPHduzevQOrV3+KkSPHmPPbExFRJQoKCsa2bd9h4cK3ERwcgl69+hUZ\nM2fOu3j//YX45psvodVq0LNnD/Tv/yI+/ngFbt68AQBo3rwlwsLCUa2aH9at+xrDh0di8OCHXwUR\nKP7M6bCwcISHRyAysh+qVfNDo0aN9e/lLS3o4+OLFSs+wcyZczB37kxkZ+dAEASMHDkGtWoFVvBP\npJg6y3oZSGM97AojcXFxeOKJlsjMzMSffx7S74O3NEq6Soq196GEHgD2YUmU0AOgjD6U0ANg4qX9\nTMnX1xcLFy5Beno6Jk0ay93MRERUZZg9dAGgV6++ePrpZ3Ho0EF8/fVqOUogIiIyO1lCVxAELF78\nP3h4eOCdd97C9evX5CiDiIjIrGQJXQDw8/PD/PmLkZ6ehsmTxxV75hkREZGSyBa6ANCv3wB06dIN\nBw7sw5o1X8lZChERUaWTNXQFQcCSJSvg7u6BuXNnVdrnooiIiCyBrKELAP7+AZg3byHS0lK5m5mI\nyEoZu7Tfnj3f4/79eDNUZJlkD10AGDBgEDp16ox9+/7Ehg1r5S6HiIjKwZil/X78cTfi4uKKfa8q\nfITUIkJXEAQsXfoBXF3d8NZbM3H79i25SyIiojLKW9pv8OD+mD17epFF4vfu/R0XLpzHvHmzMXx4\nJLKystCxY0d88smHGDHiRfz5528YN24UoqJ0l4ZMSkpE//49AegC+eOPV2DkyJcwdOgg7Nq13ez9\nmYJFhC4AVK9eA++8swApKcmYMmU8dzMTEVVA8+bOxd5MNb44MTHX0afP81i3bjOcnJywfftmg/c7\ndOiEevXqY86cd/Hll+v1a+26u3tg9eq16NSpSzFb1c2ev/9+J1xcXPH559/g88+/wa5d2xEbe6dM\n9VkCiwldABg06EV06NARv//+K779doPc5RARURkUXtrv1KmTRcZIkoTCc6pOnTqXuu2//z6Cn376\nAcOGDcIrr7yE5OQkqzz51qwLHpRGEAQsW/Yh2rV7DLNnz0CHDh3h7x8gd1lERFbn2DHjVucp7/ji\nlLa038M4OjrqH6vVakiS7thudnZ2gVESJk16HS1bPlbRMmVlUTNdAKhZsxbmzJmHpKRETJ06gbuZ\niYisRGlL+wGAs7Mz0tJSH7qNgIAauHDhHAAYLAH46KOPY9u2LdBoNACAGzdikJWVacryzcLiQhcA\nhgwZhieeaI9ffvkJW7Z8K3c5RERkhLyl/QYP7o+UlORil/Z7+ulnsWTJQv2JVIVnxy+8EInt27di\n+PDBSE5O1r/eo0cvBAfXxogRgzFkyAAsWbIQWq220nsyNbMv7WesmJjraNfuMdjZ2eLAgX/g5+dn\nosqMo6Tlpqy9DyX0ALAPS6KEHgBl9KGEHgALXtrPWIGBQZg9+20kJiZi2rRJ3M1MRERWz2JDFwCG\nDXsZrVu3xZ4932PHjq1yl0NERFQhFh26KpUK//vfSjg5OWHGjKm4d++e3CURERGVm0WHLgDUrh2C\nN9+cg4SEBMyYMVXucoiIiMrN4kMXAEaMGIVWrR7H7t07rPbSX0RERFYRuiqVCitWfAQHBwdMnz4F\n8fFVd4UKIiKyXlYRugAQEhKGGTPeQnx8PGbO5G5mIiKyPlYTugDwyitj0KLFo9ixYxt++GG33OUQ\nERGViVWFrlqtxooVH8Pe3h7Tpk1CQsJ9uUsiIiIymlWFLgCEh9fBG2/MQlzcPbz55htyl0NERGQ0\nqwtdABgzZiyaNWuOrVu/w08//Sh3OUREREaxytDV7Wb+BHZ2dnj99YlITHwgd0lERESlssrQBYCI\niLp4/fUZuHs3FrNnz5C7HCIiolJZbegCwGuvTUDjxk3x7bcb8OuvP8ldDhERUYmsOnRtbGzwwQef\nwNbWFlOnTkRSUqLcJRERET2UVYcuANSrVx+TJ0/DnTu3MWfOm3KXQ0RE9FBWH7oAMH78ZDRo0Agb\nNqzFH3/8Jnc5RERExVJE6Nra2uKDDz6BjY0NJk8eh5SUZLlLIiIiKkIRoQsADRo0xMSJU3H79i3M\nnTtb7nKIiIiKUEzoAsDEiVNRv34DrF37Ffbt+1PucoiIiAwYHbqiKKJ3794YPXp0ZdZTIXZ2dvjg\ng4+hVqsxefI4pKamyF0SERGRntGhu2bNGoSGhlZmLSbRqFETjB8/CTduxGDevDlyl0NERKRnVOjG\nxsZi37596N+/f2XXYxKTJ7+BunXr4auvvsDBg/vlLoeIiAiAkaG7YMECTJs2DYIgVHY9JmFvb48V\nKz6GSqXCxIljkZaWJndJREREsCltwN69e+Hj44N69erh6NGjRm/Y19e1QoVVVJcuHTBt2jS89957\nWLZsAT744IMyb0PuHkxFCX0ooQeAfVgSJfQAKKMPJfRgLEGSJKmkAcuWLcOuXbugVquRlZWFtLQ0\ndO7cGYsXLy5xw3Fx8p/ElJmZiaeeegIXL0Zh5849ePzxNkZ/ra+vq0X0UFFK6EMJPQDsw5IooQdA\nGX0ooQfA+P84lLp7efLkydi7dy9+//13LFu2DK1atSo1cC2Fg4MDli//CCqVChMmvIr09HS5SyIi\noipMUZ/TLU6LFo9i9OixuHbtKhYunCd3OUREVIWVKXQfffRRrFq1qrJqqTRvvPEmQkPD8NlnH+Po\n0SNyl0NERFWU4me6AODo6Ijlyz8GAEyc+CoyMjJkroiIiKqiKhG6ANCq1WN45ZUxuHz5EhYtmi93\nOUREVAVVmdAFgBkz3kJwcG2sWrUS//77t9zlEBFRFVOlQtfJyQkrVnwMURQxYcKryMzMlLskIiKq\nQqpU6ALA44+3wcsvj0J09EUsWfKe3OUQEVEVUuVCFwDefHMuAgODsXLlchw/fkzucoiIqIqokqHr\n7OyM5ctX6nczZ2VlyV0SERFVAVUydAGgbdt2GDp0BC5cOI///c86rrBFRETWrcqGLgC89dY7qFUr\nECtWLMOpUyfkLoeIiBSuSoeui4srli37EFqtFuPHv4rs7Gy5SyIiIgWr0qELAO3bP4kXXxyGc+fO\nYPnyJXKXQ0REClblQxcA5s6dhxo1amL58iU4c+a03OUQEZFCMXQBuLq6YenSD6DRaDB+/Bjk5OTI\nXRIRESkQQzdXx45PYdCgF3HmzCl8+OH/5C6HiIgUiKFbwNtvz4e/fwCWLl2E06e5m5mIiEyLoVuA\nu7sHli5dgZycHAwdOhSpqalyl0RERArC0C2kc+duiIwcgv/++w8DBvRGcnKS3CUREZFCMHSL8f77\nyzFo0CD8889R9O3bEwkJ9+UuiYiIFIChWwwbGxusWbMGgwa9iJMnj6N372cRFxcnd1lERGTlGLoP\noVarsWzZhxg+fCTOnz+LXr2exp07t+Uui4iIrBhDtwQqlQoLFy7Bq6+OR3T0RfTs2Q03bsTIXRYR\nEVkphm4pBEHAnDnzMGXKG7h+/Rp69uyGK1cuy10WERFZIYauEQRBwBtvvIlZs+bi1q2beO65pxEV\ndUHusoiIyMowdMtg/PjJmD9/Ee7ejUWvXk/j9OlTcpdERERWhKFbRiNHjsGSJSuQkJCAPn2exfHj\nx+QuiYiIrARDtxyGDBmGDz9chZSUZPTt2xNHjhyWuyQiIrICDN1yev75F/DZZ18hMzMDAwf2xoED\n++QuiYiILBxDtwJ69uyNr75aD41Gg0GD+uG3336WuyQiIrJgDN0K6tr1aaxb9x1UKhVeemkQfvhh\nt9wlERGRhWLomkCHDh2xceNW2NnZ4+WXh2Dbts1yl0RERBaIoWsirVu3xebNO+Ds7IIxY17Ghg1r\n5S6JiIgsDEPXhFq0eBTbtu2Gp6cnJk58DatXfyZ3SUREZEEYuibWqFETbN/+I3x9q2HGjKn4+OMP\n5S6JiIgsBEO3EtSrVx87d+5BQEB1zJ37JpYuXQRJkuQui4iIZMbQrSRhYeHYuXMPAgODsGjRfCxY\n8A6Dl4ioimPoVqLg4NrYuXMPQkJCsWLFUsyePZ3BS0RUhTF0K1mNGjWxc+dPqFu3Hj777BNMnToR\noijKXRYREcmAoWsGfn5+2L79RzRo0Ahr136FceNGQ6PRyF0WERGZGUPXTLy9vbFt2240b94Cmzdv\nwujRI5CTkyN3WUREZEYMXTPy8PDE5s078fjjbbBr13YMHz4YmZmZcpdFRERmwtA1MxcXV2zcuBXt\n2z+Jn3/egyFDBiI9PV3usoiIyAwYujJwcnLC2rXfokuXbti79w8MGtQPqakpcpdFRESVrNTQzc7O\nRv/+/dGrVy/06NEDK1euNEddiufg4IAvv1yHHj164dChg+jfvxeSkhLlLouIiCqRTWkD7OzssGbN\nGjg6OkKr1eKFF15Au3bt0KhRI3PUp2h2dnb49NMvYW9vjy1bvkWfPj3w3Xc74O3tLXdpRERUCYza\nvezo6AhAN+vlR11My8bGBitXfooXXxyK06dPok+f7rh7967cZRERUSUodaYLAKIook+fPoiJiUFk\nZGTps9zgYHiJRa+8lHDsTLHDvZo3KPZ1WcerhCI9VGY9XwFwGDkan3++Cr16PY2tW3ejevUaFd9+\ngT6s6s+/oNweLKaeco5HzHWLqofjOd4SxisiL4CH/v0uzKjQValU2LFjB1JTU/Hqq6/i0qVLCAsL\nK/Fr1CqhyGu+vq4P+QZFx1rC+MI9VHY9n376Mby83LFo0SL07v0M/vjjDwQHB1d4+3l9yP3nWZHx\napVgUfWUZ/xDv8ZK6i843uBrLaCe8ozXP7eQeso7vrh/a+Wsp8zjoYy8MJYglfFiwCtXroSzszOG\nDRtW4ri4OOs+G9fX11WWHiRJwtKli7B48QJUr14D27btRkhIyf/BKYlcfZiSEnoA2IclUUIPgDL6\nsPgeRBHIzISQmQEh9x4ZmRCyMiFkZgKZGRAyMuE+dJBRmyt1ppuQkABbW1u4uroiMzMThw8fxiuv\nvFLhPqh4giBg6tTpcHBwxDvvzEbPnk9jy5ZdqFu3ntylERHJy8gAzHtfN7bg89z3szLzt5M7Hpm5\n28nIHZf3fna2cbWZKnTj4uIwffp0iKIIURTxzDPPoH379sYVQeU2duwEODo6YMaM19G79zP47rsd\naNiwsdxlEREVJUlAdjaE9DQI6em5tzT9PdLTIaQ95D1JA9fEFMMAzMrSPy9XAJa1fEEAHB0hOThA\ncnCE5OICyccXkoM9JAdHIO91BwdIjo6Avb3hcwcHuBj5vUoN3YiICGzfvr2CLVF5jBgxCg4Ojpg8\neRz69OmBTZu2onnzlnKXRUTWSJKAjIwioWd4nw4UfC2taEgWHZf7ulZb7tIcCpZZXAB6+0BydCga\ngA4ORQPRwQGSfe57jo4FxjoCDvaGz/O2aWsLCGU7NluYyUKX5BUZOQQODg4YO3YU+vV7Dhs2bMbj\nj7eRuywiqkyiqAuylBQIqakQUpINH6emQJWaCkg5cI5/UCQQ9Y/T8h8jIx2CCdbzllQqSE7OkJyc\nACcniN4+kJyc9K9JTk6QnAs8dnIGDN43fM+rpi/i00VdANo7AHZ2FQ5AS8bQtQJ9+z4POzt7jB49\nHAMH9sGaNZvQvv2TcpdFRAVJku64YEpKbiimFB+aqbrHKv17KbrX8h6npEBISzU6IJ2KK8XWVh9u\nors7pIDqucH38PArGJYlhSTs7U0bir6ukCz5RCoTY+haiR49noODw3oMH/4iBg9+HqtXr0GXLk/L\nXRaR9cvJyZ095oeeKi0lPwALhmZaam5gFgzRlPyvL+fFgyQ7O0iurpBcXCEGBUN0dc197gLJxS3/\nsasrJFfc0+i4AAAgAElEQVQ3iC4ukFxc4FHTDwlZAJxzw9HRUReMtram/TMik2HoWpHOnbth/frN\nGDJkIIYOjcSnn36JHj16yV0WkbxEEUJyEoTERKiSEiEkJkJISoQqMbH415ISgdRkeCcl6YKynMtr\nSmo1JBddOIoB1SE560JRdHXLD0gXV/2Y/OB0g+iS/1hycdHNHsvD1xXaKjRLVAKGrpVp164DNm3a\nhkGD+mPkyKH48MNV6N9/oNxlEVWMkcGpSnxQJECF5KQyHauUnJwAd3eInl6QagUWmUnqQzMvLAuG\npqsrRGfdPRwdFX3skSoHQ9cKPfZYa2zZshMDBvTB2LGjkJWVhcGDX5K7LKrqSgzOB/qQzAvSigan\n6O4BsXp1iPXqQ/LwgOTuAbHQveThAdHDE5KHJ0R3D0ju7oC9PXx9XfGAM0SSAUPXSjVr1gLbtn2P\n559/DpMnj0NmZgZefnm03GWRUmRkQBUfB9X9eKjux0OIj4fq/n2o7scDmalwi40zTXB6eEKsXgNi\n/UfyQ1Iflh6FXvPUvwc7u0psnqjyMHStWMOGjbBjxx707dsDM2dOQ0ZGJsaNmyh3WWSJ0tL0AaoP\n0fgCz+/H54bsfaji43UXLShB3hFIyckZoocHg5PISAxdKxcRURe7du1B3749MW/eW8jISMfrr8+A\nwGNNyiVJhiEaHwchNyzzn+cFqm52KqSnl75Ze3uI3j7QhIVD8vaG6O2ju/n4QPLxzX3uDc/QWojX\n2up21TI4icqEoasAISFh2LlTN+NdsuQ9ZGZmYvbstxm81kKSdB9FiYszDMr4eMNdvPfv658bc8at\n5OCgC9HwiEIh6gvJx0cfonnPJWcX404MqmKfqyQyJYauQgQGBmHXrp/Qt28PrFy5HBkZ6Zg/f7Hc\nZVVdkqQ7eSg2Fqo7t6G6GwukJcL5+q1Cx0lzH2dllb5JR0eIPr7Q1K2nuwpQboDqZ6O5AZoXrnB2\n5tm1RBaGoasgAQHVsWPHHvTv/xxWr/4MWVlZ+Prr1XKXpTzp6VDF3oH6bm6g6oP1DtR37kAVeweq\nu7HFzkYLXj1IcnKG6OMDTf1HdCFaIDAfGqJEZNUYugpTrVo1bN/+PQYM6IN1677BvXt3MG/eYtSu\nHSJ3aZZPo4Hq3l1daBYIT/Wd27rHsXd0AZuU+NBNSCoVRN9qutmof4D+pg2oDrewIDywdc4PUafi\nLuBHRErG0FUgLy9vbN26C6+8Mgy//PIL9u/fjwkTpmDs2ImwL++Vb6yZJEF4kKAL0oKz0dhYqGIL\nzFTj7pX4kRfRwwNiQAA0TZvlBmkARL8AiAHVIfr76+59fAGbh/y18nWFhsdCiao0hq5Cubm5Y+PG\nrfjzzz2YMGEiFi2ajy1bvsWiRcvQrl0HucsznbQ0qO8WmJnmBqsqNm+GGgvV3TslHjOVHBwg+gcg\np9XjEIsJUq2fP0T/AN0ViIiIKoChq2CCIGDAgAFo0aINFi2aj9WrP0O/fj3Rp08/vP32Qvj5+cld\n4sPlnoikvhEDJMXB4eKV/BlqgWBVJSc9fBMqFUQ/f90xU78AXaDm7uoV/fz1wSq5e/CEIyIyC4Zu\nFeDm5o758xdjwIBBmDZtErZt24Jff/0FM2fOxtChL0OtVstTWFoa1DdioI65BlXMdaivX4c6RndT\nxVyHKiVZP9S10JeKnp4Qa9SEpnkLaP0Dip2hij6+gFy9EREVg6FbhTRq1AQ//PAb1q79GvPnv40Z\nM17Hpk0b8P77/0OTJs1M/w2zs6G6dVMfpLowvaZ7fP06VPFxxX6Z5OQEbWAQcgJbQxsYBKd6dZDs\n6gWtf26g+gcADg6mr5eIqJIxdKsYtVqNoUNH4JlneuDtt2dh8+ZN6Nr1SQwdOgIzZ74Fd3cP4zcm\nirqPzsRch+r6NYNZqjrmOlR3bkMQxSJfJtnaQluzFjSPNIA2MAjawCCIuffawGBIPj4Gu3udfF2R\nxROQiEgBGLpVVLVq1fDRR59h0KAXMW3aJHz11Rf4/vtdePvt+ejb93nd1awkCUJCAtS5s1OVfvdv\n7u7gmzcgZGcX2bYkCBADqiPn0ccKhGkQxKBg3b1/AHf7ElGVxNCt4to2boIDH32OXz/7GCd3bkPW\nqyNxcdZ0NPX0hGNsLFRpqcV+nejjkztTDS4UrEHQ1qhV/kW5iYgUjKGrdFlZUF+OLjBLzdv9mzt7\nTUgAAAzOvQEAEu4jOeE+Yn184dGmLVA7JDdYdTNVba1AwMVFro6IiKwWQ1cJJAlCfDxsoqOgvhgF\ndXQUbC5GQX0pGrh9C17FXPBBsreHtlYgNI2b5odpkC5Qf4m+iCnz38btO7cReOEC3hs6Ak891VWG\nxoiIlIWha01EEapbN2Fz8QLUFy/mh2t0FFQPHhQZrq1eA2jfHhkBNQ1OVBKDgiBW8wNUqmK/Taem\nzXHwmR5YunQRPv30Iwwa1B/du/fEu+++hxo1alZ2l0REisXQtUQ5OVBfvQL1xagCs9eLsLl0sci6\nqJJKBW1wbeS0ehza8Aho6kRAWycC2vA6kFxc4evritRynPnr4uKCOXPm4fnnX8C0aZPwww+78Oef\nv2PatJkYOXI0bG1tTdUtEVGVwdCVU1oabC5dzA/V3Fmr+uoVCBqNwVDJwQHa0HBo6tTJD9fwCGhD\nQiv1pKV69epj5849+PbbDXj77VmYO/dNfPvtBixe/D+0avVYpX1fIiIlYuiagXD/ftHjrdEXob55\no8hY0d0DmibN8kO1Th1owiMg1gqU7WM2KpUKL7wwGF27Po13352Ldeu+QY8eXRAZOQSzZ78NLy9v\nWeoiIrI2DF1TkSSobt8qsEv4ItQXL8AmOgqq+/eLDNf6+SP7iQ76UNXWiYAmPAJStWoWex1gLy9v\nLFv2IQYMiMS0aZOwfv0a7NnzPd56ax4GDoyE6iHHiImISIehW1YaDdTXrhaatUZBHR1d5DOtkkoF\nMTAIWc1bFtglXAfaOhGQ3NxlaqDiWrV6DL/9th9ffPEpFi2aj4kTX8OGDWuxePH/UL/+I3KXR0Rk\nsRi6D5OeDpvTJwuEq+5sYfWVyxBycgyGSnZ20IaGI7tAqGrCI6ANDVPsNYJtbW0xZsxYPPdcb8ya\nNR3ff78TnTq1xahRr2Hq1Olw4ed4iYiKYOgCEFKSYXPqJGxOnoDNqeOwOXkCuHIZnoU+3yq6uELT\nsBG0deoW2CVcB2JQcJW9rGH16jXw5Zdr8dtvP2P69Nfx8ccfYMeOrZg/fzGeeeZZ3eUkiYgIQBUM\nXSE5CTanT+kC9uR/uvsrlw3GiG7uQLt2yKgdVuCEpgjdNYMZIsV66qmuOHCgHVasWIIPP1yOYcMi\n0blzVyxY8D6CgoLlLo+IyCIoOnSF5KQiM9giAevugewn2kPTqAk0TZoip1ETiMG14VvNrVyfb63K\nHB0dMX36bPTtOwBvvDEZv/76Mw4e3I9Jk17Hq6+Oh52dndwlEhHJSjGhW6aAbdwUmsZN9AHL2atp\nhYfXwdatu7F163eYM+dNLFjwDjZv3oRFi5ahbdt2cpdHRCQbqwzdIgF74jhsrl4xGKML2A7QNG7C\ngJWBIAjo128AOnfuioUL5+Grr75Anz7Pol+/AZg7dz6qVasmd4lERGZn8aGrD9gTx/NnsKUFbOOm\nupObGLCyc3f3wHvvLcWAAYMwbdpkbNnyLX755Se8+eYcDBkyDOoqegIaEVVNFhW6QlJi0V3EJQRs\nTpOm0DRqwoC1Ak2bNsdPP/2Br79ejQUL3sEbb0zGpk3r8P77y9GoURO5yyMiMgvZQteogPXwQHa7\nJ3Nnr00YsFZOrVZjxIhX8OyzPTFnzkxs27YFXbp0wPDhIzF9+iy4WfEFQ4iIjGGW0DUI2JPHYXvy\nONTXrhqMYcBWHX5+/li16ku88MKLmD59Cr744lPs2rUD8+YtRK9effnZXiJSrMoJ3T/+gOPev2Bz\n6oRxAdu4KcTAIAZsFdO+/ZPYu/cwVq5cjuXLl2DUqOFYv34tFi1agtDQcLnLIyIyucoJ3U6dkHcR\nQIOAzTsGy4ClXPb29pgy5Q306dMfM2ZMxR9//Ib27R/HuHGTMGHCFDgo9DKaRFQ1VU7oTp+OpPD6\nDFgyWu3aIdi4cSu+/34XZs16A0uXLsLWrd/lnvncW+7yiIhMotS12GJjYzFkyBA888wz6NGjB9as\nWVP6VhcuRHaPXjwmS2UiCAJ69HgOf/31D0aNeg03bsRg4MA+6NevH44d+wdSoWthExFZm1JDV61W\nY8aMGfjxxx+xadMmrF+/HpcvXy7ty4jKzcXFFfPmLcSvv+5HixaPYuvWrXj66U7o0OFxfPbZx0hI\nKLo+MRGRNSg1dH19fVGvXj0AgLOzM0JDQ3Hv3r1KL4yoQYOG+P77X/DTTz+hZ8/euHQpGrNmTUej\nRhEYNWoY9u/fC1EU5S6TiMhoZTqme/PmTVy4cAGNGjWqrHqIDKhUKnTt2hXNmrVGfHw8Nm/ehHXr\nvsb27VuxfftWBAUFIzJyCAYOjIS/f4Dc5RIRlUiQjDxQlpaWhhdffBGvvvoqnnrqqRLHBgej2BnI\nsWNpxY5v3ty52NflHK9SqYr0YE315ynYhyXUU57xeT3kjZckCX//fRTr13+DXbu2Iz39LADAwcER\nLi4ucHBwgCAIFlN/npgYFeKKWbnK0v/8C4/39XU16EPuesozvmAPllBPecf7+roiMLD4vT3WUD8A\ntGzpavV5Aej+fhvDqJmuRqPB+PHj8dxzz5UauHlUqqIF+Pq6PmRs8duQe3zhHuSup7zj8/qwlHrK\nM16lUhmMf/bZznj22c5ISkpCSIgKqakpyMzMQGZmBtRqNVxcXJCUdB9hYWEWUX9JX2MNf/6Fxxd8\nbAn1lGd83nNLqaf844v/AmupX/c11p8XxjJqpjtt2jR4enpixowZRm+4uP/RW5PC/5u3Vkrow9ge\nTp8+hQ0b1mDLlu+QlJQIAGjbth0iI4ege/eesn/mVwk/C0AZfSihB0AZfSihB6Dk/1QUVGpGHzt2\nDLt378aRI0fQq1cv9O7dG/v3769wgUSm1rBhIyxcuASnTkXh448/R5s2T+Dgwf0YM+ZlNGpUBzNn\nvo6zZ8/IXSYRVWFGH9MtK2v/n4uS/vdl7X1UpIcrVy5hw4Z12LhxHeLidGfdN2vWHJGRL6F3775w\ncTHuf6emoISfBaCMPpTQA6CMPpTQA2DCmS6RNQsJCcOsWXNx4sR5fPPNRnTp0g0nThzHlCnj0aBB\nHUyc+Br++ecoL7xBRGbB0KUqwdbWFk8/3R3r1n2H48fPYcaM2fDx8cWGDWvRvXtntGvXCqtWrcT9\n+7zwBhFVHoYuVTkBAdUxadLr+PvvE9i8eSd69eqDq1ev4K23ZqJRozoYOXIo9u79gxfeICKTk20R\neyK5qVQqtG//JNq3fxL379/Hli2bsH79GuzcuQ07d25DYGAQXnhhMF54YTCqV68hd7lEVMmys4H0\ndCA9XShwr3uclpb/WkZG0TEbNxr3PXgi1UMo6eC+tfdhzh4kScKxY/9g/fo12L59K9LT06BSqdCx\n41OIjHwJXbp0g62tbbm2rYSfBaCMPpTQA6CMPsrSgyiimMDLv8/IKDksC79XOEA1mvIv0GNsknKm\nS1SAIAho0eJRtGjxKObNW4gdO7Zh/fpv8Ntvv+C3336Br281DBgwCIMHD0FISNELbxCRjiQBaWlA\naqqAlBQBKSmGj9PSdI+1WiA+3v6hQVg4LE1BpZLg5AQ4OenuvbzEAs91rzk7S3B0zB9jeF/0NehX\nkS8ZZ7oPoYT/QQLK6MMSejh37iw2bFiDzZs34cGDBwCA1q3bIjJyCJ599jk4OjqWug1L6MMUlNCH\nEnoATN+HJAFZWXnhmB+SqanIDUvd4/zwzH+vuK+RpPKHpL19ySFX8N7R0XCMs/PDw9LREbC3N/2q\ns8Z+ZIih+xD8S2k5LKmHzMxM7NnzPdatW4MDB/YCANzc3NGv3/OIjHwJDRs+fDEQS+qjIpTQhxJ6\nAPL70GhQKAx1j4ubZRYOybzHea/n5JQvjezsJLi6SnBxAVxcdI9dXQFXVwnOzvmPdWN0z11cJNSs\n6YTs7LQiM0y12sR/WJWMoVtBSvtLac0stYdr165i48a12LhxPWJj7wAAGjduisjIIejTpx/c3NwN\nxltqH2WlhD4srQdJ0h2rTEwUkJgoICkp7x548CD/eeH309JUSE6Wyr3bVRAMw9DZuWAwokBA5j/P\nC1NdkOaHp719+Xq3tJ9FeTF0K0hJvwjW3oel96DRaPD7779i/fpv8OuvP0Or1cLR0RE9e/ZGZORL\naNXqMQiCYPF9GEsJfVRWD5mZKBSQMAjJwqFZ8P3sbOOD09ZWgru7BC8vFRwdtUVmjwXDMO/1ggGa\n956Tk+l3s5aVEn6fAIZuhSnpF8Ha+7CmHmJj7+Dbbzdg/fo1uHbtKgAgLCwckZEvYeTIobCzc5O5\nwoqzpp/Hw5TUQ04ODELz4YFZ9P3MTOMTTK2W4OEhwd0d8PCQ9Dd3d6nQ86Lv54Wl0n8W1oShW0FK\n+kWw9j6ssQdRFHHo0EGsW/cNfvhhF7KysgAAoaFhaN26rf4WEFBd5krLzlp+Hnlnz8bHC7h/X0B8\nvID4eBXu3xeQnm6PO3dy9KFZcBduWXbVCoIuFN3dJXh65gem4fOi73t46HbXVnSWaS0/i5IooQeA\noVthSvpFsPY+rL2HBw8SsG3bZhw48Cf27z+A1NT8XkJCQvUB3KbNE1YRwnL+PDIzDUM0Li7vsapQ\nuOoeZ2QYl2pubg+bZepC82GzUFfXsq+nakrW/ncDUEYPAEO3wpT0i2DtfSihB0DXx507D3DmzCn8\n9ddBHDp0AEeOHEZKSrJ+TO3aIWjT5gk8/ngbtGnzhEVeCcuUP4+cHCAhQReexYWmLlhV+sepqaWH\nqL29BB8fw5u3twQfH1H/PDTUCZKUCg8PCW5ugI2VXrFACX83lNADwNCtMCX9Ilh7H0roASi+D61W\naxDChw8fMgjh4ODaBiFco0ZNc5ddREk/D61Wd7Zt4QDNn5EWfE+FxMTSQ9TGJi808wPU17domOa9\n7uxc+m5bJf9OWRsl9AAwdCtMSb8I1t6HEnoAjOtDq9Xi7NnTBiGcnJykfz8oKNgghGvWrFXZZUOj\nAe7dE3DnjoDYWBUyMx1x7VpWkRlpfLyAhAQBolhy4qlUEry8Cs9Ciz7OC1N398q5kEFV+Z2ydEro\nAWDoVpiSfhGsvQ8l9ACUrw+tVotz587gr78O4NChgzh8+BCSkhL17wcGBqNNm/wTs2rVCizT9tPS\ngNhYAbdvq/SheueOgNu38x/fu1d6kHp46EKy5Bmp7ubpKcl+4YOq/DtlaZTQA8DQrTAl/SJYex9K\n6AEwTR+6ED6LQ4cO4K+/DuLw4b8KhXAQWrdui8cea4v69dtDrQ7MDVEVYmMF3LmjC1LdTYXk5IeH\nqZ2dBH9/CQEBIgICJAQESPD3FxEW5gBb23T4+OhC1ctLQjnXgJANf6cshxJ6AIwPXSs9fYCoalKr\n1ahTpxHc3BqjcePx6NVLwokT93DqVDyuXMnErVu22LTJD5s21QBg99DtuLtLqFFDRPPmulD195dQ\nvXr+44AA3ey0uN26vr4OiIvTVl6TRArG0CWyEJIEJCejwK5e3Wy04K7e2FjdCUiGaufedMdLfXyy\n4eAQj5yca0hMPI2srCsAbgG4CT8/EW3ahKB9+1Zo3botAgODIMh9SSKiKoShS2QGWi1w6xZw5oyq\nwK7egrt7da+VdGEGJyfdDLRuXU3uzFTM3eWrm6FWr67b3as7XuoKoCFE8RGcP38Ohw8fxF9/peDw\n4YPYtu0Atm37BgBQo0ZN/WeEW7dui6CgYIYwUSXiMd2HUNJxBmvvw1p6SEkBrl9X4do1Fa5fF3D9\nukr//ObNkldv8fExPG4aEKAL1bxdvQEBItzcKn4WryiKuHDhfG4IH8Thwwdx//59/fs1atTUnxnd\nunVbBAfXLhLC1vLzKIkSegCU0YcSegB4IlWFKekXwdr7sJQetFrdmb7Fher16wLu3y/+0kQ+PiKC\ngiSEhqrh5ZVtcGJSQIAIP7/yr9BSUaIoIirqAg4dOoBDh/7CoUMHDEK4evUaBiFcu3YIqlVzs4if\nR0VYyu9URSmhDyX0ADB0K0xJvwjW3oc5e0hNhT5M84JVF6oq3LhR/EowtrYSAgMlBAWJ+ltwcP5z\nFxfz91FekiQhKuoC/vrrAA4f1oVwfHy8/n1//wA0bdoEgYEhqFMnAuHhEQgPrwNvb28Zqy47a/hZ\nGEMJfSihB4BnLxMVSxR1s9W8UL12LT9Ur18v7iQlHW9vEQ0aFAxV3ew1KEg3a5X7c6emIggC6tat\nh7p162HEiFcgSRIuXozSh/CRI4ewZ8+eIl/n7e2tD+D8WwRq1qwFlZwXJyayMAxdUpz0dBjMVAvu\nAo6JUSErq+hs1cZGQq1aEho00BQJ1aAg3fHUqkgQBERE1EVERF0MHz4SAGBjo8GRI/8hOvoiLl6M\nwqVLuvu//z6CI0cOGXy9o6MjQkPDUadOnQKhHIGQkFDYy7VPnUhGDF2yOpIE3L1reGy14Gz13r3i\nZ1aenhLq1Ss6Uw0K0p35a60XvTc3T09PtGjxKFq0eNTg9czMTFy9egXR0VEFwvgiLl+OxpkzpwzG\nqlQqBAUFo06dCISF1cndVa2bIbu7e5izHSKz4j8zZJEkSbcb+MIFFWJjgTNn7PWhGhOjKnbJNrVa\nQs2aEtq10+hDVXevu7m7y9BIFeLg4IB69eqjXr36Bq+LooibN2/khvFF/cw4OjoKP/+8Bz//bLi7\nulo1v9wwDjc4bhwQUJ0fZyKrx9AlWUmS7mL6Fy6oEBWlu124oMbFiyokJRX8B1Z3dSU3Nwnh4WKB\nMJX0M9caNThbtUQqlQqBgUEIDAxCp05dDN67f/8+oqOj9Luqo6OjcOlSNA4e3I+DB/cbjHV2dkF4\neDjCwyMMZsjBwbVha23XoaQqi/9EkdnExeWHa37Iqoss76ZWSwgJEfHEEyIiIkS0bGkPb+80BAWJ\n8OCeR0Xx9vaGt3drPPZYa4PX09PTcflydIEw1s2Qz507ixMnjhuMtbGxQe3aIQXCOFx/7+Ji3Bml\nRObC0CWTu39fKBSsulvhz7GqVBJq15bQurUGdevqAjYiQkRoqGjwuVVfX3vExYlm7oLk5OTkhIYN\nG6Nhw8YGr2s0GsTEXEN0dLTBSVzR0RcRHX0RP/6422B89eo1DM6mzpsh+/i4mLMdIj2GLpXbgwdA\nVJS60K5hVZGP3QiChKAgCS1b5hiEa1iYCAcHmYonq2RjY4OQkDCEhISha9en9a9LkoR79+4VOYkr\nOjoK+/b9iX37/jTYjouLC/z9A+DvHwA/P//cez+D1/z8/OHk5GTuFknhGLpUqqQk4MIFtUGwRkWp\nij1LODBQRJcuGkREaBERIaJuXV248t8uqkyCIMDPzw9+fn5o27adwXupqSkFPt6kmyHfvHkdt2/f\nxqVL0SVu193dA/7+/vDzC4C/v39uKPvnhnL+Y378iYzF0CW9lBTkBqraIFxjY4uGa61aIp56SpM7\na9Wibl0R4eEinJ1lKJyoBC4urmjatDmaNm2ufy3vKkhZWVm4d+8u7t6NRWxsLO7evYPY2FjExt5B\nbOyd3NfvICrqQonfw8vLq5hgDjAI6WrV/HjCFzF0q6LUVODixfwzhfPC9fbtouFao4aIjh01ubNW\n3ey1Tp38SxsSWTN7e3vUqhWIWrUCSxyXkZGBe/fuFgjm/HDOC+Zbt27i/PmzD92GIAjw9vbRB3HB\nXdsFw9nHxxc2PA1fsfiTVbD0dODff4HDh230s9eoKBVu3CgargEBIjp00Oh3CeftHnblyZ9EcHR0\nRFBQMIKCgkscl5aWhrt3Y/VBnB/M+Y+vXLlc5GIhBalUKvj6Vis0Yy46g7a2612TDkNXITQa3a7h\n48fVOH5chf/+081gRREAHPXjqlUT8cQT+WcL581eeeEIoopzdnZGSEgoQkJCSxyXmppisBu74K7t\n/F3a53Hy5PGHbsPGxgZeXl5wc3OHu7s73N09Ctx76J97eHjAza3ovVopFwy3MgxdKyRJQEyMgOPH\n1fjvP13InjqlNrhKk5OThJYttWjZ0gaBgZn62aunp4yFExEA3XHmsDBXhIWFP3SMJElITk4q9hjz\n3bt3ERt7B8nJiUhIeICYmOvIzs4uUw2urm7FhLW7QVjnv+ZpEOCOjo68Olg5MXStQEICcOJEXsDq\nQrbgx3JUKgl164po1kyLZs1ENG2qm73a2OSdMJIjY/VEVB6CIOhnrBERdYsdk3dCmCRJyMjIQHJy\nEhITE5GUlISkpAe597rniYmJ+vcL3sfEXEdKSnKZarOzs9PPmjnLLhuGroXJyABOn87bTawL2mvX\nDI/BBgaKeO65HDRtqgvZhg21PGuYqAoTBAFOTk5wcnKCv39Amb9eq9UiOTnJIKSTkhILBHhigZth\nkF+/fg05OWX7j72rq5s+gH18vGBraw9HR139jo6OcHJy1t87ORV87lRgXP69s7Pu3hrCnKErI60W\niI5W4b//VPpZ7PnzKmg0+bttPD0ldOyoyQ1YLZo0EeHrK8lYNREpjVqthqenFzw9vcr8tWWdZRcM\n7piY6zh79rTJ+rC3ty8S2oXDOu/2sJA3DHvDcLezs6vwbnWGrplIEnD7tqA/Bnv8uBonTqiRlpb/\nA7S3l9CkiW43cdOmulvt2hJ46ISILFVFZ9ne3s6IibmH9PR0ZGSkF3ufd8vIyEB6elqh+4LjdK+l\npaUjOTkZsbGxyMhIhyia5jKyarW6SFjnzcT3799r1DZKDd2ZM2di79698Pb2xu7du0sbTrmSkqDf\nRZx3NnHBKzgJgoSICBFNm4r6WWzduiLs7GQsmojIzFQqFZydneFcScfIJElCVlZWgSDXBXZ6enEB\nXtmry4cAAAsRSURBVFyQFwx9w/vExESkp6eVafd6qaHbp08fvPjii5g2bVqFGleyrCzg7FmVwdnE\nly4ZHluoXl1E9+45aNpUN5Nt3FjLz8ASEVUyQRDg4OAABweHcu0+N4ZJQ7dFixa4detWhQpSElEE\nLl/WHYfNm8meOaNCTk7+PmBXV91C6rrdxLqZrL8/j8MSESlRWS7vyWO6pbh7V3ccNu9kpxMn1EhJ\nyQ9YW1sJDRqI+mOwzZrplqZTFb3oExERVXEM3QIkCTh/XoUDB9Q4fhw4csS5yPWIw8K06NYt/2Sn\nRx4xXPuViIjoYSotdH19reOA5Y0bwG+/6W6//w7cvZv/np+fCj17Ao8+CrRqBbRoAXh4qAGoAVjP\naiHW8rMoiRJ6ANiHJVFCD4Ay+lBCD8YyKnQlqezHI+PiUsr8NeaQlAQcPGiD/fvV2L/fBpcv589k\nq1UT0a+fFu3aadCzpyMcHVMMPq6TkwPExclQdAXkXbHGmimhB4B9WBIl9AAoow8l9AAY/x+HUkN3\nypQpOHr0KBITE9GhQweMGzcOffv2rXCB5pKVBfzzj1ofsidOqCCKuiR1dpbQpYsG7dpp0K6d7tKJ\neSHr62t9AUtERJat1NBdunSpOeowGVHUfXxn3z5dyB49mr8QgI2NbhGAdu10t2bNtOCa0kREZC6K\nOJHq+nUB+/frdhkfOKBGQkL+LuN69fJCVoPHH9dy8XUiIpKNVYZuQoLuuGzebPb69fyQrV5dxMCB\nOWjXToMnntDCz4+fjyUiIstgFaGbkQEcPZp/XPb0aRUkSbfL2M1NwjPP5Ohns6GhvFYxERFZJosM\nXa0WOHVKpd9l/PffamRl6ZLUzk5Cmzb5u4wbNdKtG0tERGTpLCKuJAm4elXAvn26kD140AZJSfnT\n1YYN80O2VSstnJxkLJaIiKicZAvde/cEHDyYv8v45s3847KBgSJ69tTtMm7TRgsfHx6XJSIi62e2\n0E1NBY4cUetns+fP56/C4+kp6UO2XTsNgoMZskREpDyVFro5OcDx4/nHZf/9Vw2NRrfL2MFBQvv2\nugtStG+vQYMGXCCAiIiUr1JCt2dP4M8/XZCaqgtZQZDQpImov/JTy5ZaODhUxncmIiKyXJUSurt3\nAyEhEvr10+0ybttWAw+PyvhORERE1qNSQvfaNcDJKa0yNk1ERGS1KuVIalBQZWyViIjIuvH0JSIi\nIjNh6BIREZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIR\nEZkJQ5eIiMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eI\niMhMGLpERERmwtAlIiIyE4YuERGRmTB0iYiIzIShS0REZCYMXSIiIjNh6BIREZkJQ5eIiMhMGLpE\nRERmwtAlIiIyE4YuERGRmRgVuvv370e3bt3QtWtXfPbZZ5VdExERkSKVGrqiKGLevHlYvXo1vv/+\ne/zwww+4fPmyOWojIiJSlFJD99SpUwgKCkKNGjVga2uL7t274/fffzdHbURERIpSaujevXsXAQEB\n+ud+fn64d+9epRZFRESkRKWGriRJ5qiDiIhI8WxKG+Dv74/bt2/rn9+9exfVqlUrdcO+vq4Vq8wC\nKKEHQBl9KKEHgH1YEiX0ACijDyX0YKxSZ7oNGzZETEwMbt26hezsbPzwww/o1KmTOWojIiJSlFJn\numq1GrNnz8bw4cMhSRL69euH0NBQc9RGRESkKILEg7ZERERmwStSERERmQlDl4iIyEwYukRERGZS\n6olUZbF//34sWLAAkiShb9++eOWVV0y5ebOYOXMm9u7dC29vb+zevVvucsolNjYW06ZNQ3x8PNRq\nNfr3748hQ4bIXVaZZWdnIzIyEjk5OdBqtejatSvGjh0rd1nlIooi+vbtCz8/P6xatUrucsqlY8eO\ncHFxgUqlgo2NDbZs2SJ3SeWSkpKCN998E9HR0VCpVFiwYAEaN24sd1lGu3r1KiZNmgRBECBJEm7c\nuIEJEyZY5d/xr7/+Glu2bIEgCKhTpw4W/r+9u3mJag8DOP6dHKRQexElCyzIjCySFr1AEyamSTXV\nxGCLNiVRbdIow14oghYJLfoHWkREEBEaRG1EszGmQiuGYIgwIhhMKkRT5yXPnOcu4l64G+89x7nz\na7rPZz1n+A6HmYcznHmmo4P8/HzTWY7cunXrr/fCv/qslQxJp9NSX18vsVhMfvz4IXv37pWhoaFM\nPX3WDAwMSDQaFb/fbzrFtS9fvkg0GhURkcnJSdmxY0dOngsRkXg8LiIilmVJU1OTRCIRw0Xu3Lx5\nU9ra2uT48eOmU1yrq6uTsbEx0xmzdvbsWbl//76IiExPT8vExIThIvfS6bT4fD4ZHh42neLYyMiI\n1NXVSSqVEhGRkydPSldXl+EqZ96/fy9+v19SqZRYliWHDx+WT58+zXhMxr5e/l12NG/YsIH58+eb\nzpiV0tJSqqqqACgoKKCioiJnV3fOmzcP+HnVa1mW4Rp3RkZGePr0KU1NTaZTZkVEsG3bdMasTE5O\nMjg4SDAYBMDr9VJYWGi4yr1wOMyyZcv+tqo3l9i2TSKRwLIsksnkv1q89Cv58OED69evJz8/n7y8\nPDZu3Eh3d/eMx2Rs6OqO5l9TLBbj3bt3VFdXm05xxbZtAoEAPp8Pn8+Xk6/j6tWrtLe34/F4TKfM\nisfj4ciRIwSDQe7du2c6x5VYLMaiRYs4f/48+/fv59KlSySTSdNZrj1+/Jjdu3ebznBl8eLFNDc3\nU1tbS01NDUVFRWzZssV0liOVlZUMDAwwPj5OIpEgFArx+fPnGY/J2NAV/bnvL2dqaorW1lYuXLhA\nQUGB6RxX5syZw4MHDwiFQkQiEYaGhkwnOdLX10dJSQlVVVU5/x65e/cunZ2d3Lhxgzt37jA4OGg6\nyTHLsohGoxw8eJCuri7mzp2bs/8RPj09TW9vLzt37jSd4sr379/p6enhyZMn9Pf3E4/Hc+4+moqK\nCo4ePUpzczPHjh1j9erVeL0z3yqVsaHrdkez+m9YlkVrayv79u2jvr7edM6sFRYWsmnTJvr7+02n\nOPL69Wt6e3vZvn07bW1tvHz5kvb2dtNZrpSWlgJQXFxMQ0MDb9++NVzkXFlZGWVlZaxbtw6AxsZG\notGo4Sp3QqEQa9eupbi42HSKK+FwmPLychYuXEheXh4NDQ28efPGdJZjwWCQzs5Obt++zYIFC1i+\nfPmMj8/Y0P2ddjTn+hUJ/LwLe+XKlRw6dMh0imujo6NMTEwAkEwmef78OStWrDBc5czp06fp6+uj\np6eH69evs3nzZq5du2Y6y7FEIsHU1BQA8XicZ8+eUVlZabjKuZKSEpYsWcLHjx8BePHiRc6utX30\n6BF+v990hmtLly4lEomQSqUQkZw9F6OjowAMDw/T3d39j+ckYz8Z+l12NP95NTI2NkZtbS0tLS1/\n3XSRK169esXDhw9ZtWoVgUAAj8fDqVOnqKmpMZ3myNevXzl37hy2bWPbNrt27WLbtm2ms/6Xvn37\nxokTJ/B4PKTTafbs2cPWrVtNZ7ly8eJFzpw5g2VZlJeX09HRYTrJsWQySTgc5sqVK6ZTXKuurqax\nsZFAIIDX62XNmjUcOHDAdJZjLS0tjI+P4/V6uXz5MkVFM/9jku5eVkoppbJEN1IppZRSWaJDVyml\nlMoSHbpKKaVUlujQVUoppbJEh65SSimVJTp0lVJKqSzRoauUUkpliQ5dpZRSKkv+AO2e4yf8wTuC\nAAAAAElFTkSuQmCC\n", + "text/plain": [ + "\u003cmatplotlib.figure.Figure at 0xc1dc310\u003e" + ] + }, + "metadata": { + "tags": [] + }, + "output_type": "display_data" + } + ], + "source": [ + "# Train our variables.\n", + "\n", + "# numpy is used for its asscalar() function.\n", + "import numpy as np\n", + "\n", + "num_training_steps = 10\n", + "\n", + "def train_model(inputs, labels, wb, optimizer, num_training_steps):\n", + " loss_at_step = []\n", + " w_at_step = []\n", + " b_at_step = []\n", + " for step_num in range(num_training_steps):\n", + " loss, gradients_and_variables = value_and_gradients_fn(inputs, labels, wb)\n", + " loss_at_step.append(np.asscalar(loss.numpy()))\n", + " \n", + " optimizer.apply_gradients(gradients_and_variables)\n", + " w, b = wb.variables\n", + " w_at_step.append(np.asscalar(w.read_value().numpy()))\n", + " b_at_step.append(np.asscalar(b.read_value().numpy()))\n", + "\n", + " print(w_at_step)\n", + " t = range(0, num_training_steps)\n", + " plt.plot(t, loss_at_step, 'k',\n", + " t, w_at_step, 'r',\n", + " t, [true_w] * num_training_steps, 'r--',\n", + " t, b_at_step, 'b',\n", + " t, [true_b] * num_training_steps, 'b--')\n", + " plt.legend(['loss', 'w estimate', 'w true', 'b estimate', 'b true'])\n", + " plt.show()\n", + "\n", + "train_model(inputs, labels, wb, optimizer, num_training_steps)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "UNurY9VJ-hpH" + }, + "source": [ + "## Other Ways to Compute Gradients\n", + "\n", + "Using our loss function as an example (`calculate_linear_model_loss()`), there are several other ways we could compute gradients:\n", + "\n", + "1. `tfe.implicit_gradients()`\n", + "1. `tfe.gradients_function()`\n", + "1. `tfe.implicit_value_and_gradients()`\n", + "1. `tfe.value_and_gradients_function()`\n", + "\n", + "Each of these functions does the following:\n", + "* Wraps a function.\n", + "* Returns a function with the same input signature as the wrapped function.\n", + "\n", + "They differ only in what information they return.\n", + "\n", + "### Gradients-only functions\n", + "\n", + "The following two functions return a function that returns only the variables' gradients:\n", + "\n", + "1. `tfe.gradients_function()`: Returns the partial derivatives of the function `f()` with respect to the parameters of `f()`.\n", + "1. `tfe.implicit_gradients()`: Returns the partial derivatives of the function `f()` with respect to the trainable parameters (`tf.Variable`) used by `f()`.\n", + "\n", + "In our example above, the `tf.layers.Dense` object encapsulates the trainable parameters.\n", + "\n", + "### Value and gradients functions\n", + "\n", + "The following two functions are identical to their counterparts above, except that they also return the value of the wrapped function.\n", + "\n", + "1. `tfe.implicit_value_and_gradients()`\n", + "1. `tfe.value_and_gradients_function()`\n", + "\n", + "### Gradient demos\n", + "\n", + "In the demos below, we show examples for the `implicit_*` functions, since our existing loss function works seamlessly with these versions. (The other versions require that your parameters are tensors and tensors only; in our example, we're using a `Dense` layer.)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 85, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 100, + "status": "ok", + "timestamp": 1505502831671, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "aEoCftnfAIH5", + "outputId": "72f1c1dc-a574-463f-f860-c4e5f48fcdaa" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[(\u003ctf.Tensor: id=673, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", + " (\u003ctf.Tensor: id=671, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)]" + ] + }, + "execution_count": 13, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# tfe.implicit_gradients() demo\n", + "gradients_fn = tfe.implicit_gradients(loss_fn)\n", + "\n", + "# Returns only gradients and variables:\n", + "gradients_fn(inputs, labels, wb)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 102, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 88, + "status": "ok", + "timestamp": 1505502831785, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 240 + }, + "id": "bbgCUdCzAVhH", + "outputId": "152aa9b6-9e42-4b7e-848a-9423c0b1929c" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(\u003ctf.Tensor: id=688, shape=(), dtype=float32, numpy=1.0623235\u003e,\n", + " [(\u003ctf.Tensor: id=720, shape=(1, 1), dtype=float32, numpy=array([[-0.26846504]], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/kernel:0' shape=(1, 1) dtype=float32\u003e),\n", + " (\u003ctf.Tensor: id=718, shape=(1,), dtype=float32, numpy=array([-0.32890949], dtype=float32)\u003e,\n", + " \u003ctf.Variable 'dense/bias:0' shape=(1,) dtype=float32\u003e)])" + ] + }, + "execution_count": 14, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# tfe.implicit_value_and_gradients() demo\n", + "value_gradients_fn = tfe.implicit_value_and_gradients(loss_fn)\n", + "\n", + "# Returns only gradients:\n", + "value_gradients_fn(inputs, labels, wb)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Eager Execution Tutorial: Working with Gradients", + "provenance": [], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..ebcc7027c1d34c47a339a49ede1d80e58ad43780 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/notebooks/3_datasets.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "U9i2Dsh-ziXr" + }, + "source": [ + "# Eager Execution Tutorial: Importing Data\n", + "\n", + "This notebook demonstrates the use of the [`tf.contrib.data.Dataset` API](https://www.tensorflow.org/programmers_guide/datasets) to build pipelines to feed data to your program. It covers:\n", + "\n", + "* Creating a `Dataset`.\n", + "* Iteration over a `Dataset` with eager execution enabled.\n", + "\n", + "We recommend using the `Dataset`s API for building performant, complex input pipelines from simple, re-usable pieces that will feed your model's training or evaluation loops.\n", + "\n", + "If you're familiar with TensorFlow graphs, the API for constructing the `Dataset` object remains exactly the same when eager execution is enabled, but the process of iterating over elements of the dataset is slightly different. You will use a Pythonic `Iterator()` class instead of using `make_one_shot_iterator()` and `get_next()`. As a result, the discussion on iterators in the [Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is not relevant when eager execution is enabled." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "z1JcS5iBXMRO" + }, + "source": [ + "# Setup: Enable eager execution\n" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "RlIWhyeLoYnG" + }, + "outputs": [], + "source": [ + "# Import TensorFlow.\n", + "import tensorflow as tf\n", + "\n", + "# Import TensorFlow eager execution support (subject to future changes).\n", + "import tensorflow.contrib.eager as tfe\n", + "\n", + "# Enable eager execution\n", + "tfe.enable_eager_execution()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "H9UySOPLXdaw" + }, + "source": [ + "# Step 1: Create a source `Dataset`\n", + "\n", + "Create a _source_ dataset using one of the factory functions like [`Dataset.from_tensors`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensors), [`Dataset.from_tensor_slices`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#from_tensor_slices) or using objects that read from files like [`TextLineDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TextLineDataset) or [`TFRecordDataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/TFRecordDataset). See the [Programmer's Guide](https://www.google.com/url?sa=D\u0026q=https%3A%2F%2Fwww.tensorflow.org%2Fprogrammers_guide%2Fdatasets%23reading_input_data) for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "WPTUfGq6kJ5w" + }, + "outputs": [], + "source": [ + "ds_tensors = tf.contrib.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])\n", + "\n", + "# Create a CSV file\n", + "import tempfile\n", + "_, filename = tempfile.mkstemp()\n", + "with open(filename, 'w') as f:\n", + " f.write(\"\"\"Line 1\n", + "Line 2\n", + "Line 3\n", + " \"\"\")\n", + "ds_file = tf.contrib.data.TextLineDataset(filename)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "twBfWd5xyu_d" + }, + "source": [ + "# Step 2: Apply transformations\n", + "\n", + "Use the transformations functions like [`map`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#map), [`batch`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#batch), [`shuffle`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset#shuffle) etc. to apply transformations to the records of the dataset. See the [API documentation for `tf.contrib.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/contrib/data/Dataset) for details." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "cellView": "code", + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + } + }, + "colab_type": "code", + "id": "ngUe237Wt48W" + }, + "outputs": [], + "source": [ + "ds_tensors = ds_tensors.map(tf.square).shuffle(2).batch(2)\n", + "ds_file = ds_file.batch(2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "IDY4WsYRhP81" + }, + "source": [ + "# Step 3: Iterate\n", + "\n", + "Use `tfe.Iterator` on the `Dataset` object to get a Python iterator over the contents of the dataset.\n", + "\n", + "If you're familiar with the use of `Dataset`s in TensorFlow graphs, note that this process of iteration is different. Here there are no calls to `Dataset.make_one_shot_iterator()` and no `get_next()` calls." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "autoexec": { + "startup": false, + "wait_interval": 0 + }, + "height": 153, + "output_extras": [ + { + "item_id": 1 + } + ] + }, + "colab_type": "code", + "executionInfo": { + "elapsed": 201, + "status": "ok", + "timestamp": 1505952405928, + "user": { + "displayName": "", + "photoUrl": "", + "userId": "" + }, + "user_tz": 420 + }, + "id": "lCUWzso6mbqR", + "outputId": "ec027d30-96c6-4ea4-9ee1-ef74ec1ae29a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Elements of ds_tensors:\n", + "tf.Tensor([4 9], shape=(2,), dtype=int32)\n", + "tf.Tensor([16 25], shape=(2,), dtype=int32)\n", + "tf.Tensor([36 1], shape=(2,), dtype=int32)\n", + "\n", + "Elements in ds_file:\n", + "tf.Tensor(['Line 1' 'Line 2'], shape=(2,), dtype=string)\n", + "tf.Tensor(['Line 3' ' '], shape=(2,), dtype=string)\n" + ] + } + ], + "source": [ + "print('Elements of ds_tensors:')\n", + "for x in tfe.Iterator(ds_tensors):\n", + " print(x)\n", + "\n", + "print('\\nElements in ds_file:')\n", + "for x in tfe.Iterator(ds_file):\n", + " print(x)" + ] + } + ], + "metadata": { + "colab": { + "default_view": {}, + "last_runtime": { + "build_target": "", + "kind": "local" + }, + "name": "Eager Execution Tutorial: Importing Data", + "provenance": [], + "version": "0.3.2", + "views": {} + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tensorflow/contrib/eager/python/examples/resnet50/BUILD b/tensorflow/contrib/eager/python/examples/resnet50/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..5759ca17facda2e94a35bcc7e2a54b80ff5ac858 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/BUILD @@ -0,0 +1,43 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_library( + name = "resnet50", + srcs = ["resnet50.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + ], +) + +cuda_py_test( + name = "resnet50_test", + size = "large", + srcs = ["resnet50_test.py"], + additional_deps = [ + ":resnet50", + "//tensorflow/contrib/summary:summary_test_util", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) + +cuda_py_test( + name = "resnet50_graph_test", + size = "large", + srcs = ["resnet50_graph_test.py"], + additional_deps = [ + ":resnet50", + "//tensorflow/contrib/summary:summary_test_util", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + ], + tags = [ + "noasan", + "nomsan", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/resnet50/README.md b/tensorflow/contrib/eager/python/examples/resnet50/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f6c1defa4246d46447028f86c87c4ea9b39bb2ad --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/README.md @@ -0,0 +1,34 @@ +Image classification using the ResNet50 model described in +[Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385). + +Contents: + +- `resnet50.py`: Model definition +- `resnet50_test.py`: Sanity unittests and benchmarks for using the model with + eager execution enabled. +- `resnet50_graph_test.py`: Sanity unittests and benchmarks when using the same + model code to construct a TensorFlow graph. + +# Benchmarks + +Using a synthetic data. + +``` +# Using eager execution +bazel run -c opt --config=cuda :resnet50_test -- --benchmarks=. + +# Using graph execution +bazel run -c opt --config=cuda :resnet50_graph_test -- --benchmarks=. +``` + +(Or remove the `--config=cuda` flag for running on CPU instead of GPU). + +On October 31, 2017, the benchmarks demostrated comparable performance +for eager and graph execution of this particular model when using +a single NVIDIA Titan X (Pascal) GPU on a host with an +Intel Xeon E5-1650 CPU @ 3.50GHz and a batch size of 32. + +| Benchmark name | batch size | images/second | +| --------------------------------------- | ------------- | ------------- | +| eager_train_gpu_batch_32_channels_first | 32 | 171 | +| graph_train_gpu_batch_32_channels_first | 32 | 172 | diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py new file mode 100644 index 0000000000000000000000000000000000000000..b302a87e0e8a61d2456db1eba847f31bd70f552e --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py @@ -0,0 +1,324 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""ResNet50 model definition compatible with TensorFlow's eager execution. + +Reference [Deep Residual Learning for Image +Recognition](https://arxiv.org/abs/1512.03385) + +Adapted from tf.keras.applications.ResNet50. A notable difference is that the +model here outputs logits while the Keras model outputs probability. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import tensorflow as tf +import tensorflow.contrib.eager as tfe + + +class _IdentityBlock(tfe.Network): + """_IdentityBlock is the block that has no conv layer at shortcut. + + Args: + kernel_size: the kernel size of middle conv layer at main path + filters: list of integers, the filters of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + data_format: data_format for the input ('channels_first' or + 'channels_last'). + """ + + def __init__(self, kernel_size, filters, stage, block, data_format): + super(_IdentityBlock, self).__init__(name='') + filters1, filters2, filters3 = filters + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + bn_axis = 1 if data_format == 'channels_first' else 3 + + self.conv2a = self.track_layer( + tf.layers.Conv2D( + filters1, (1, 1), + name=conv_name_base + '2a', + data_format=data_format)) + self.bn2a = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')) + + self.conv2b = self.track_layer( + tf.layers.Conv2D( + filters2, + kernel_size, + padding='same', + data_format=data_format, + name=conv_name_base + '2b')) + self.bn2b = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')) + + self.conv2c = self.track_layer( + tf.layers.Conv2D( + filters3, (1, 1), + name=conv_name_base + '2c', + data_format=data_format)) + self.bn2c = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')) + + def call(self, input_tensor, training=False): + x = self.conv2a(input_tensor) + x = self.bn2a(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2b(x) + x = self.bn2b(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2c(x) + x = self.bn2c(x, training=training) + + x += input_tensor + return tf.nn.relu(x) + + +class _ConvBlock(tfe.Network): + """_ConvBlock is the block that has a conv layer at shortcut. + + Args: + kernel_size: the kernel size of middle conv layer at main path + filters: list of integers, the filterss of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + data_format: data_format for the input ('channels_first' or + 'channels_last'). + strides: strides for the convolution. Note that from stage 3, the first + conv layer at main path is with strides=(2,2), and the shortcut should + have strides=(2,2) as well. + """ + + def __init__(self, + kernel_size, + filters, + stage, + block, + data_format, + strides=(2, 2)): + super(_ConvBlock, self).__init__(name='') + filters1, filters2, filters3 = filters + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + bn_axis = 1 if data_format == 'channels_first' else 3 + + self.conv2a = self.track_layer( + tf.layers.Conv2D( + filters1, (1, 1), + strides=strides, + name=conv_name_base + '2a', + data_format=data_format)) + self.bn2a = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')) + + self.conv2b = self.track_layer( + tf.layers.Conv2D( + filters2, + kernel_size, + padding='same', + name=conv_name_base + '2b', + data_format=data_format)) + self.bn2b = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')) + + self.conv2c = self.track_layer( + tf.layers.Conv2D( + filters3, (1, 1), + name=conv_name_base + '2c', + data_format=data_format)) + self.bn2c = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')) + + self.conv_shortcut = self.track_layer( + tf.layers.Conv2D( + filters3, (1, 1), + strides=strides, + name=conv_name_base + '1', + data_format=data_format)) + self.bn_shortcut = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name=bn_name_base + '1')) + + def call(self, input_tensor, training=False): + x = self.conv2a(input_tensor) + x = self.bn2a(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2b(x) + x = self.bn2b(x, training=training) + x = tf.nn.relu(x) + + x = self.conv2c(x) + x = self.bn2c(x, training=training) + + shortcut = self.conv_shortcut(input_tensor) + shortcut = self.bn_shortcut(shortcut, training=training) + + x += shortcut + return tf.nn.relu(x) + + +class ResNet50(tfe.Network): + """Instantiates the ResNet50 architecture. + + Args: + data_format: format for the image. Either 'channels_first' or + 'channels_last'. 'channels_first' is typically faster on GPUs while + 'channels_last' is typically faster on CPUs. See + https://www.tensorflow.org/performance/performance_guide#data_formats + name: Prefix applied to names of variables created in the model. + trainable: Is the model trainable? If true, performs backward + and optimization after call() method. + include_top: whether to include the fully-connected layer at the top of the + network. + pooling: Optional pooling mode for feature extraction when `include_top` + is `False`. + - `None` means that the output of the model will be the 4D tensor + output of the last convolutional layer. + - `avg` means that global average pooling will be applied to the output of + the last convolutional layer, and thus the output of the model will be + a 2D tensor. + - `max` means that global max pooling will be applied. + classes: optional number of classes to classify images into, only to be + specified if `include_top` is True. + + Raises: + ValueError: in case of invalid argument for data_format. + """ + + def __init__(self, + data_format, + name=None, + trainable=True, + include_top=True, + pooling=None, + classes=1000): + super(ResNet50, self).__init__(name='') + + valid_channel_values = ('channels_first', 'channels_last') + if data_format not in valid_channel_values: + raise ValueError('Unknown data_format: %s. Valid values: %s' % + (data_format, valid_channel_values)) + self.include_top = include_top + + def conv_block(filters, stage, block, strides=(2, 2)): + l = _ConvBlock( + 3, + filters, + stage=stage, + block=block, + data_format=data_format, + strides=strides) + return self.track_layer(l) + + def id_block(filters, stage, block): + l = _IdentityBlock( + 3, filters, stage=stage, block=block, data_format=data_format) + return self.track_layer(l) + + self.conv1 = self.track_layer( + tf.layers.Conv2D( + 64, (7, 7), + strides=(2, 2), + data_format=data_format, + padding='same', + name='conv1')) + bn_axis = 1 if data_format == 'channels_first' else 3 + self.bn_conv1 = self.track_layer( + tf.layers.BatchNormalization(axis=bn_axis, name='bn_conv1')) + self.max_pool = self.track_layer( + tf.layers.MaxPooling2D((3, 3), strides=(2, 2), data_format=data_format)) + + self.l2a = conv_block([64, 64, 256], stage=2, block='a', strides=(1, 1)) + self.l2b = id_block([64, 64, 256], stage=2, block='b') + self.l2c = id_block([64, 64, 256], stage=2, block='c') + + self.l3a = conv_block([128, 128, 512], stage=3, block='a') + self.l3b = id_block([128, 128, 512], stage=3, block='b') + self.l3c = id_block([128, 128, 512], stage=3, block='c') + self.l3d = id_block([128, 128, 512], stage=3, block='d') + + self.l4a = conv_block([256, 256, 1024], stage=4, block='a') + self.l4b = id_block([256, 256, 1024], stage=4, block='b') + self.l4c = id_block([256, 256, 1024], stage=4, block='c') + self.l4d = id_block([256, 256, 1024], stage=4, block='d') + self.l4e = id_block([256, 256, 1024], stage=4, block='e') + self.l4f = id_block([256, 256, 1024], stage=4, block='f') + + self.l5a = conv_block([512, 512, 2048], stage=5, block='a') + self.l5b = id_block([512, 512, 2048], stage=5, block='b') + self.l5c = id_block([512, 512, 2048], stage=5, block='c') + + self.avg_pool = self.track_layer( + tf.layers.AveragePooling2D( + (7, 7), strides=(7, 7), data_format=data_format)) + + if self.include_top: + self.fc1000 = self.track_layer( + tf.layers.Dense(classes, name='fc1000')) + else: + reduction_indices = [1, 2] if data_format == 'channels_last' else [2, 3] + reduction_indices = tf.constant(reduction_indices) + if pooling == 'avg': + self.global_pooling = functools.partial( + tf.reduce_mean, + reduction_indices=reduction_indices, + keep_dims=False) + elif pooling == 'max': + self.global_pooling = functools.partial( + tf.reduce_max, reduction_indices=reduction_indices, keep_dims=False) + else: + self.global_pooling = None + + def call(self, input_tensor, training=False): + x = self.conv1(input_tensor) + x = self.bn_conv1(x, training=training) + x = tf.nn.relu(x) + x = self.max_pool(x) + + x = self.l2a(x, training=training) + x = self.l2b(x, training=training) + x = self.l2c(x, training=training) + + x = self.l3a(x, training=training) + x = self.l3b(x, training=training) + x = self.l3c(x, training=training) + x = self.l3d(x, training=training) + + x = self.l4a(x, training=training) + x = self.l4b(x, training=training) + x = self.l4c(x, training=training) + x = self.l4d(x, training=training) + x = self.l4e(x, training=training) + x = self.l4f(x, training=training) + + x = self.l5a(x, training=training) + x = self.l5b(x, training=training) + x = self.l5c(x, training=training) + + x = self.avg_pool(x) + + if self.include_top: + return self.fc1000(tf.layers.flatten(x)) + elif self.global_pooling: + return self.global_pooling(x) + else: + return x diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py new file mode 100644 index 0000000000000000000000000000000000000000..736a75332ff6403ea1b21387211df6b8fb6034f3 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_graph_test.py @@ -0,0 +1,163 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests and benchmarks for ResNet50 under graph execution.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tempfile +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 +from tensorflow.contrib.summary import summary_test_util + + +def data_format(): + return 'channels_first' if tf.test.is_gpu_available() else 'channels_last' + + +def image_shape(batch_size): + if data_format() == 'channels_first': + return [batch_size, 3, 224, 224] + return [batch_size, 224, 224, 3] + + +def random_batch(batch_size): + images = np.random.rand(*image_shape(batch_size)).astype(np.float32) + num_classes = 1000 + labels = np.random.randint( + low=0, high=num_classes, size=[batch_size]).astype(np.int32) + one_hot = np.zeros((batch_size, num_classes)).astype(np.float32) + one_hot[np.arange(batch_size), labels] = 1. + return images, one_hot + + +class ResNet50GraphTest(tf.test.TestCase): + + def testApply(self): + batch_size = 64 + with tf.Graph().as_default(): + images = tf.placeholder(tf.float32, image_shape(None)) + model = resnet50.ResNet50(data_format()) + predictions = model(images) + + init = tf.global_variables_initializer() + + with tf.Session() as sess: + sess.run(init) + np_images, _ = random_batch(batch_size) + out = sess.run(predictions, feed_dict={images: np_images}) + self.assertAllEqual([64, 1000], out.shape) + + def testTrainWithSummary(self): + with tf.Graph().as_default(): + images = tf.placeholder(tf.float32, image_shape(None), name='images') + labels = tf.placeholder(tf.float32, [None, 1000], name='labels') + + tf.train.get_or_create_global_step() + logdir = tempfile.mkdtemp() + with tf.contrib.summary.always_record_summaries(): + with tf.contrib.summary.create_summary_file_writer( + logdir, max_queue=0, + name='t0').as_default(): + model = resnet50.ResNet50(data_format()) + logits = model(images, training=True) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels) + tf.contrib.summary.scalar(name='loss', tensor=loss) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + train_op = optimizer.minimize(loss) + + init = tf.global_variables_initializer() + self.assertEqual(321, len(tf.global_variables())) + + batch_size = 32 + with tf.Session() as sess: + sess.run(init) + sess.run(tf.contrib.summary.summary_writer_initializer_op()) + np_images, np_labels = random_batch(batch_size) + sess.run([train_op, tf.contrib.summary.all_summary_ops()], + feed_dict={images: np_images, labels: np_labels}) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].tag, 'loss') + + +class ResNet50Benchmarks(tf.test.Benchmark): + + def _report(self, label, start, num_iters, batch_size): + avg_time = (time.time() - start) / num_iters + dev = 'gpu' if tf.test.is_gpu_available() else 'cpu' + name = 'graph_%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format()) + extras = {'examples_per_sec': batch_size / avg_time} + self.report_benchmark( + iters=num_iters, wall_time=avg_time, name=name, extras=extras) + + def benchmark_graph_apply(self): + with tf.Graph().as_default(): + images = tf.placeholder(tf.float32, image_shape(None)) + model = resnet50.ResNet50(data_format()) + predictions = model(images) + + init = tf.global_variables_initializer() + + batch_size = 64 + with tf.Session() as sess: + sess.run(init) + np_images, _ = random_batch(batch_size) + num_burn, num_iters = (3, 30) + for _ in range(num_burn): + sess.run(predictions, feed_dict={images: np_images}) + start = time.time() + for _ in range(num_iters): + # Comparison with the eager execution benchmark in resnet50_test.py + # isn't entirely fair as the time here includes the cost of copying + # the feeds from CPU memory to GPU. + sess.run(predictions, feed_dict={images: np_images}) + self._report('apply', start, num_iters, batch_size) + + def benchmark_graph_train(self): + for batch_size in [16, 32, 64]: + with tf.Graph().as_default(): + np_images, np_labels = random_batch(batch_size) + dataset = tf.data.Dataset.from_tensors((np_images, np_labels)).repeat() + (images, labels) = dataset.make_one_shot_iterator().get_next() + + model = resnet50.ResNet50(data_format()) + logits = model(images, training=True) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + train_op = optimizer.minimize(loss) + + init = tf.global_variables_initializer() + with tf.Session() as sess: + sess.run(init) + (num_burn, num_iters) = (5, 10) + for _ in range(num_burn): + sess.run(train_op) + start = time.time() + for _ in range(num_iters): + sess.run(train_op) + self._report('train', start, num_iters, batch_size) + + +if __name__ == '__main__': + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py new file mode 100644 index 0000000000000000000000000000000000000000..d6389f2e385b3637b178d49fc56e8baf913eccaa --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py @@ -0,0 +1,234 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests and benchmarks for the ResNet50 model, executed eagerly.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import tempfile +import time + +import tensorflow as tf + +import tensorflow.contrib.eager as tfe +from tensorflow.contrib.eager.python.examples.resnet50 import resnet50 +from tensorflow.contrib.summary import summary_test_util +from tensorflow.python.client import device_lib + + +def device_and_data_format(): + return ('/gpu:0', 'channels_first') if tfe.num_gpus() else ('/cpu:0', + 'channels_last') + + +def random_batch(batch_size): + _, data_format = device_and_data_format() + + shape = (3, 224, 224) if data_format == 'channels_first' else (224, 224, 3) + shape = (batch_size,) + shape + + num_classes = 1000 + images = tf.random_uniform(shape) + labels = tf.random_uniform( + [batch_size], minval=0, maxval=num_classes, dtype=tf.int32) + one_hot = tf.one_hot(labels, num_classes) + + return images, one_hot + + +def train_one_step(model, images, labels, optimizer): + + def model_loss(): + logits = model(images, training=True) + loss = tf.losses.softmax_cross_entropy( + logits=logits, onehot_labels=labels) + tf.contrib.summary.scalar(name='loss', tensor=loss) + return loss + + optimizer.minimize(model_loss) + + +class ResNet50Test(tf.test.TestCase): + + def test_apply(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + with tf.device(device): + images, _ = random_batch(2) + output = model(images) + self.assertEqual((2, 1000), output.shape) + + def test_apply_no_top(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format, include_top=False) + with tf.device(device): + images, _ = random_batch(2) + output = model(images) + output_shape = ((2, 2048, 1, 1) + if data_format == 'channels_first' else (2, 1, 1, 2048)) + self.assertEqual(output_shape, output.shape) + + def test_apply_with_pooling(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format, include_top=False, pooling='avg') + with tf.device(device): + images, _ = random_batch(2) + output = model(images) + self.assertEqual((2, 2048), output.shape) + + def test_train(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + tf.train.get_or_create_global_step() + logdir = tempfile.mkdtemp() + with tf.contrib.summary.create_summary_file_writer( + logdir, max_queue=0, + name='t0').as_default(), tf.contrib.summary.always_record_summaries(): + with tf.device(device): + optimizer = tf.train.GradientDescentOptimizer(0.1) + images, labels = random_batch(2) + train_one_step(model, images, labels, optimizer) + self.assertEqual(320, len(model.variables)) + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].tag, 'loss') + + def test_no_garbage(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + optimizer = tf.train.GradientDescentOptimizer(0.1) + with tf.device(device): + images, labels = random_batch(2) + gc.disable() + # Warm up. Note that this first run does create significant amounts of + # garbage to be collected. The hope is that this is a build-only effect, + # and a subsequent training loop will create nothing which needs to be + # collected. + train_one_step(model, images, labels, optimizer) + gc.collect() + previous_gc_debug_flags = gc.get_debug() + gc.set_debug(gc.DEBUG_SAVEALL) + for _ in range(2): + # Run twice to ensure that garbage that is created on the first + # iteration is no longer accessible. + train_one_step(model, images, labels, optimizer) + gc.collect() + # There should be no garbage requiring collection. + self.assertEqual(0, len(gc.garbage)) + gc.set_debug(previous_gc_debug_flags) + gc.enable() + + +class MockIterator(object): + + def __init__(self, tensors): + self._tensors = [tf.identity(x) for x in tensors] + + def next(self): + return self._tensors + + +class ResNet50Benchmarks(tf.test.Benchmark): + + def _train_batch_sizes(self): + """Choose batch sizes based on GPU capability.""" + for device in device_lib.list_local_devices(): + if 'GPU:0' in device.name: + # Avoid OOM errors with larger batch sizes, which seem to cause errors + # later on even if caught. + # + # TODO(allenl): Base this on device memory; memory limit information + # during the test seems to exclude the amount TensorFlow has allocated, + # which isn't useful. + if 'K20' in device.physical_device_desc: + return (16,) + if 'P100' in device.physical_device_desc: + return (16, 32, 64) + return (16, 32) + + def _report(self, label, start, num_iters, device, batch_size, data_format): + avg_time = (time.time() - start) / num_iters + dev = 'cpu' if 'cpu' in device else 'gpu' + name = '%s_%s_batch_%d_%s' % (label, dev, batch_size, data_format) + extras = {'examples_per_sec': batch_size / avg_time} + self.report_benchmark( + iters=num_iters, wall_time=avg_time, name=name, extras=extras) + + def _force_gpu_sync(self): + # If this function is called in the context of a GPU device + # (e.g., inside a 'with tf.device("/gpu:0")' block) + # then this will force a copy from CPU->GPU->CPU, which forces + # a sync. This is a roundabout way, yes. + tf.constant(1.).cpu() + + def benchmark_eager_apply(self): + device, data_format = device_and_data_format() + model = resnet50.ResNet50(data_format) + batch_size = 64 + num_burn = 5 + num_iters = 30 + with tf.device(device): + images, _ = random_batch(batch_size) + for _ in xrange(num_burn): + model(images).cpu() + gc.collect() + start = time.time() + for _ in xrange(num_iters): + model(images).cpu() + self._report('eager_apply', start, num_iters, device, batch_size, + data_format) + + def _benchmark_eager_train(self, label, make_iterator): + device, data_format = device_and_data_format() + for batch_size in self._train_batch_sizes(): + (images, labels) = random_batch(batch_size) + num_burn = 3 + num_iters = 10 + model = resnet50.ResNet50(data_format) + optimizer = tf.train.GradientDescentOptimizer(0.1) + + with tf.device(device): + iterator = make_iterator((images, labels)) + for _ in xrange(num_burn): + (images, labels) = iterator.next() + train_one_step(model, images, labels, optimizer) + self._force_gpu_sync() + gc.collect() + + start = time.time() + for _ in xrange(num_iters): + (images, labels) = iterator.next() + train_one_step(model, images, labels, optimizer) + self._force_gpu_sync() + self._report(label, start, num_iters, device, batch_size, data_format) + + def benchmark_eager_train(self): + self._benchmark_eager_train('eager_train', MockIterator) + + def benchmark_eager_train_datasets(self): + + def make_iterator(tensors): + with tf.device('/device:CPU:0'): + ds = tf.data.Dataset.from_tensors(tensors).repeat() + return tfe.Iterator(ds) + + self._benchmark_eager_train('eager_train_dataset', make_iterator) + + +if __name__ == '__main__': + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..b657d31f35bafd6624ac7e4d6a6f6b2db362649d --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/BUILD @@ -0,0 +1,26 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "rnn_colorbot", + srcs = ["rnn_colorbot.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + "@six_archive//:six", + ], +) + +cuda_py_test( + name = "rnn_colorbot_test", + srcs = ["rnn_colorbot_test.py"], + additional_deps = [ + ":rnn_colorbot", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md b/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fabd7b3e206d3a1954893a2b75361146d4709d00 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/README.md @@ -0,0 +1,26 @@ +RNN Colorbot: An RNN that predicts colors using eager execution. + +To train and generate colors, run: + +``` +python rnn_colorbot.py +``` + +This example shows how to: + 1. read, process, (one-hot) encode, and pad text data via the + Datasets API; + 2. build a trainable model; + 3. implement a multi-layer RNN using Python control flow + constructs (e.g., a for loop); + 4. train a model using an iterative gradient-based method; and + 5. log training and evaluation loss for consumption by TensorBoard + (to view summaries, use: tensorboard --log_dir=/summaries). + +The data used in this example is licensed under the Creative Commons +Attribution-ShareAlike License and is available at + https://en.wikipedia.org/wiki/List_of_colors:_A-F + https://en.wikipedia.org/wiki/List_of_colors:_G-M + https://en.wikipedia.org/wiki/List_of_colors:_N-Z + +This example was adapted from + https://github.com/random-forests/tensorflow-workshop/tree/master/extras/colorbot diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py new file mode 100644 index 0000000000000000000000000000000000000000..318962c634e0d050b35da5efc405400380c1b759 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py @@ -0,0 +1,338 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""TensorFlow Eager Execution Example: RNN Colorbot. + +This example builds, trains, and evaluates a multi-layer RNN that can be +run with eager execution enabled. The RNN is trained to map color names to +their RGB values: it takes as input a one-hot encoded character sequence and +outputs a three-tuple (R, G, B) (scaled by 1/255). + +For example, say we'd like the RNN Colorbot to generate the RGB values for the +color white. To represent our query in a form that the Colorbot could +understand, we would create a sequence of five 256-long vectors encoding the +ASCII values of the characters in "white". The first vector in our sequence +would be 0 everywhere except for the ord("w")-th position, where it would be +1, the second vector would be 0 everywhere except for the +ord("h")-th position, where it would be 1, and similarly for the remaining three +vectors. We refer to such indicator vectors as "one-hot encodings" of +characters. After consuming these vectors, a well-trained Colorbot would output +the three tuple (1, 1, 1), since the RGB values for white are (255, 255, 255). +We are of course free to ask the colorbot to generate colors for any string we'd +like, such as "steel gray," "tensorflow orange," or "green apple," though +your mileage may vary as your queries increase in creativity. + +This example shows how to: + 1. read, process, (one-hot) encode, and pad text data via the + Datasets API; + 2. build a trainable model; + 3. implement a multi-layer RNN using Python control flow + constructs (e.g., a for loop); + 4. train a model using an iterative gradient-based method; and + +The data used in this example is licensed under the Creative Commons +Attribution-ShareAlike License and is available at + https://en.wikipedia.org/wiki/List_of_colors:_A-F + https://en.wikipedia.org/wiki/List_of_colors:_G-M + https://en.wikipedia.org/wiki/List_of_colors:_N-Z + +This example was adapted from + https://github.com/random-forests/tensorflow-workshop/tree/master/extras/colorbot +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import functools +import os +import sys +import time + +import six +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.python.eager import context + +try: + import matplotlib.pyplot as plt # pylint: disable=g-import-not-at-top + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + + +def parse(line): + """Parse a line from the colors dataset.""" + + # Each line of the dataset is comma-separated and formatted as + # color_name, r, g, b + # so `items` is a list [color_name, r, g, b]. + items = tf.string_split([line], ",").values + rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255. + # Represent the color name as a one-hot encoded character sequence. + color_name = items[0] + chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256) + # The sequence length is needed by our RNN. + length = tf.cast(tf.shape(chars)[0], dtype=tf.int64) + return rgb, chars, length + + +def load_dataset(data_dir, url, batch_size): + """Loads the colors data at path into a PaddedDataset.""" + + # Downloads data at url into data_dir/basename(url). The dataset has a header + # row (color_name, r, g, b) followed by comma-separated lines. + path = tf.contrib.learn.datasets.base.maybe_download( + os.path.basename(url), data_dir, url) + + # This chain of commands loads our data by: + # 1. skipping the header; (.skip(1)) + # 2. parsing the subsequent lines; (.map(parse)) + # 3. shuffling the data; (.shuffle(...)) + # 3. grouping the data into padded batches (.padded_batch(...)). + dataset = tf.data.TextLineDataset(path).skip(1).map(parse).shuffle( + buffer_size=10000).padded_batch( + batch_size, padded_shapes=([None], [None, None], [])) + return dataset + + +# pylint: disable=not-callable +class RNNColorbot(tfe.Network): + """Multi-layer (LSTM) RNN that regresses on real-valued vector labels. + """ + + def __init__(self, rnn_cell_sizes, label_dimension, keep_prob): + """Constructs an RNNColorbot. + + Args: + rnn_cell_sizes: list of integers denoting the size of each LSTM cell in + the RNN; rnn_cell_sizes[i] is the size of the i-th layer cell + label_dimension: the length of the labels on which to regress + keep_prob: (1 - dropout probability); dropout is applied to the outputs of + each LSTM layer + """ + super(RNNColorbot, self).__init__(name="") + self.label_dimension = label_dimension + self.keep_prob = keep_prob + + # Note the calls to `track_layer` below; these calls register the layers as + # network components that house trainable variables. + self.cells = [ + self.track_layer(tf.nn.rnn_cell.BasicLSTMCell(size)) + for size in rnn_cell_sizes + ] + self.relu = self.track_layer( + tf.layers.Dense(label_dimension, activation=tf.nn.relu, name="relu")) + + def call(self, chars, sequence_length, training=False): + """Implements the RNN logic and prediction generation. + + Args: + chars: a Tensor of dimension [batch_size, time_steps, 256] holding a + batch of one-hot encoded color names + sequence_length: a Tensor of dimension [batch_size] holding the length + of each character sequence (i.e., color name) + training: whether the invocation is happening during training + + Returns: + A tensor of dimension [batch_size, label_dimension] that is produced by + passing chars through a multi-layer RNN and applying a ReLU to the final + hidden state. + """ + # Transpose the first and second dimensions so that chars is of shape + # [time_steps, batch_size, dimension]. + chars = tf.transpose(chars, [1, 0, 2]) + # The outer loop cycles through the layers of the RNN; the inner loop + # executes the time steps for a particular layer. + batch_size = int(chars.shape[1]) + for l in range(len(self.cells)): + cell = self.cells[l] + outputs = [] + state = cell.zero_state(batch_size, tf.float32) + # Unstack the inputs to obtain a list of batches, one for each time step. + chars = tf.unstack(chars, axis=0) + for ch in chars: + output, state = cell(ch, state) + outputs.append(output) + # The outputs of this layer are the inputs of the subsequent layer. + chars = tf.stack(outputs, axis=0) + if training: + chars = tf.nn.dropout(chars, self.keep_prob) + # Extract the correct output (i.e., hidden state) for each example. All the + # character sequences in this batch were padded to the same fixed length so + # that they could be easily fed through the above RNN loop. The + # `sequence_length` vector tells us the true lengths of the character + # sequences, letting us obtain for each sequence the hidden state that was + # generated by its non-padding characters. + batch_range = [i for i in range(batch_size)] + indices = tf.stack([sequence_length - 1, batch_range], axis=1) + hidden_states = tf.gather_nd(chars, indices) + return self.relu(hidden_states) + + +def loss(labels, predictions): + """Computes mean squared loss.""" + return tf.reduce_mean(tf.square(predictions - labels)) + + +def test(model, eval_data): + """Computes the average loss on eval_data, which should be a Dataset.""" + avg_loss = tfe.metrics.Mean("loss") + for (labels, chars, sequence_length) in tfe.Iterator(eval_data): + predictions = model(chars, sequence_length, training=False) + avg_loss(loss(labels, predictions)) + print("eval/loss: %.6f\n" % avg_loss.result()) + with tf.contrib.summary.always_record_summaries(): + tf.contrib.summary.scalar("loss", avg_loss.result()) + + +def train_one_epoch(model, optimizer, train_data, log_interval=10): + """Trains model on train_data using optimizer.""" + + tf.train.get_or_create_global_step() + + def model_loss(labels, chars, sequence_length): + predictions = model(chars, sequence_length, training=True) + loss_value = loss(labels, predictions) + tf.contrib.summary.scalar("loss", loss_value) + return loss_value + + for (batch, (labels, chars, sequence_length)) in enumerate( + tfe.Iterator(train_data)): + with tf.contrib.summary.record_summaries_every_n_global_steps(log_interval): + batch_model_loss = functools.partial(model_loss, labels, chars, + sequence_length) + optimizer.minimize( + batch_model_loss, global_step=tf.train.get_global_step()) + if log_interval and batch % log_interval == 0: + print("train/batch #%d\tloss: %.6f" % (batch, batch_model_loss())) + + +SOURCE_TRAIN_URL = "https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv" +SOURCE_TEST_URL = "https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv" + + +def main(_): + data_dir = os.path.join(FLAGS.dir, "data") + train_data = load_dataset( + data_dir=data_dir, url=SOURCE_TRAIN_URL, batch_size=FLAGS.batch_size) + eval_data = load_dataset( + data_dir=data_dir, url=SOURCE_TEST_URL, batch_size=FLAGS.batch_size) + + model = RNNColorbot( + rnn_cell_sizes=FLAGS.rnn_cell_sizes, + label_dimension=3, + keep_prob=FLAGS.keep_probability) + optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) + + if FLAGS.no_gpu or tfe.num_gpus() <= 0: + print(tfe.num_gpus()) + device = "/cpu:0" + else: + device = "/gpu:0" + print("Using device %s." % device) + + log_dir = os.path.join(FLAGS.dir, "summaries") + tf.gfile.MakeDirs(log_dir) + train_summary_writer = tf.contrib.summary.create_summary_file_writer( + os.path.join(log_dir, "train"), flush_secs=10) + test_summary_writer = tf.contrib.summary.create_summary_file_writer( + os.path.join(log_dir, "eval"), flush_secs=10, name="eval") + + with tf.device(device): + for epoch in range(FLAGS.num_epochs): + start = time.time() + with train_summary_writer.as_default(): + train_one_epoch(model, optimizer, train_data, FLAGS.log_interval) + end = time.time() + print("train/time for epoch #%d: %.2f" % (epoch, end - start)) + with test_summary_writer.as_default(): + test(model, eval_data) + + print("Colorbot is ready to generate colors!") + while True: + try: + color_name = six.moves.input( + "Give me a color name (or press enter to exit): ") + except EOFError: + return + + if not color_name: + return + + _, chars, length = parse(color_name) + with tf.device(device): + (chars, length) = (tf.identity(chars), tf.identity(length)) + chars = tf.expand_dims(chars, 0) + length = tf.expand_dims(length, 0) + preds = tf.unstack(model(chars, length, training=False)[0]) + + # Predictions cannot be negative, as they are generated by a ReLU layer; + # they may, however, be greater than 1. + clipped_preds = tuple(min(float(p), 1.0) for p in preds) + rgb = tuple(int(p * 255) for p in clipped_preds) + print("rgb:", rgb) + data = [[clipped_preds]] + if HAS_MATPLOTLIB: + plt.imshow(data) + plt.title(color_name) + plt.show() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--dir", + type=str, + default="/tmp/rnn_colorbot/", + help="Directory to download data files and save logs.") + parser.add_argument( + "--log_interval", + type=int, + default=10, + metavar="N", + help="Log training loss every log_interval batches.") + parser.add_argument( + "--num_epochs", type=int, default=20, help="Number of epochs to train.") + parser.add_argument( + "--rnn_cell_sizes", + type=int, + nargs="+", + default=[256, 128], + help="List of sizes for each layer of the RNN.") + parser.add_argument( + "--batch_size", + type=int, + default=64, + help="Batch size for training and eval.") + parser.add_argument( + "--keep_probability", + type=float, + default=0.5, + help="Keep probability for dropout between layers.") + parser.add_argument( + "--learning_rate", + type=float, + default=0.01, + help="Learning rate to be used during training.") + parser.add_argument( + "--no_gpu", + action="store_true", + default=False, + help="Disables GPU usage even if a GPU is available.") + + FLAGS, unparsed = parser.parse_known_args() + tfe.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py new file mode 100644 index 0000000000000000000000000000000000000000..75b342ba78bd5de5c2827296f6fba01ffa86d560 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot_test.py @@ -0,0 +1,71 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.contrib.eager.python.examples.rnn_colorbot import rnn_colorbot + + +LABEL_DIMENSION = 5 + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" + + +def random_dataset(): + batch_size = 64 + time_steps = 10 + alphabet = 50 + chars = tf.one_hot( + tf.random_uniform( + [batch_size, time_steps], minval=0, maxval=alphabet, dtype=tf.int32), + alphabet) + sequence_length = tf.constant( + [time_steps for _ in range(batch_size)], dtype=tf.int64) + labels = tf.random_normal([batch_size, LABEL_DIMENSION]) + return tf.data.Dataset.from_tensors((labels, chars, sequence_length)) + + +class RNNColorbotTest(tf.test.TestCase): + + def testTrainOneEpoch(self): + model = rnn_colorbot.RNNColorbot( + rnn_cell_sizes=[256, 128, 64], + label_dimension=LABEL_DIMENSION, + keep_prob=1.0) + optimizer = tf.train.AdamOptimizer(learning_rate=.01) + dataset = random_dataset() + with tf.device(device()): + rnn_colorbot.train_one_epoch(model, optimizer, dataset) + + def testTest(self): + model = rnn_colorbot.RNNColorbot( + rnn_cell_sizes=[256], + label_dimension=LABEL_DIMENSION, + keep_prob=1.0) + dataset = random_dataset() + with tf.device(device()): + rnn_colorbot.test(model, dataset) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..db2587bf2cb548ae37e58597691e96ae2c2e8177 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/BUILD @@ -0,0 +1,35 @@ +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//tensorflow:internal"]) + +load("//tensorflow:tensorflow.bzl", "cuda_py_test") + +py_binary( + name = "rnn_ptb", + srcs = ["rnn_ptb.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/contrib/eager/python:tfe", + ], +) + +cuda_py_test( + name = "rnn_ptb_test", + srcs = ["rnn_ptb_test.py"], + additional_deps = [ + ":rnn_ptb", + "//tensorflow/contrib/eager/python:tfe", + "//tensorflow:tensorflow_py", + ], +) + +cuda_py_test( + name = "rnn_ptb_graph_test", + srcs = ["rnn_ptb_graph_test.py"], + additional_deps = [ + ":rnn_ptb", + "//third_party/py/numpy", + "//tensorflow:tensorflow_py", + ], +) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md b/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ea92d59e5863226a1bc28a07919518f209587cb5 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/README.md @@ -0,0 +1,42 @@ +Recurrent Neural Network model. + +Implements a language modeling network described in +https://www.tensorflow.org/tutorials/recurrent +that is compatible with (and idiomatic for) eager execution. + +To run: + +- Download and extract the Penn Treebank dataset from + http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz + + ```sh + tar xvzf simple-examples.tgz -C /tmp + ``` + +- Run: `python rnn_ptb.py --data-dir=/tmp/simple-examples/data` + + +Benchmarks (using synthetic data): + +``` +# Using eager execution +bazel run -c opt --config=cuda :rnn_ptb_test -- --benchmarks=. + +# Using graph execution +bazel run -c opt --config=cuda :rnn_ptb_graph_test -- --benchmarks=. +``` + +(Or remove the `--config=cuda` flag for running on CPU instead of GPU). + +On October 31, 2017, the benchmarks demostrated slightly better performance +(3-6%) for graph execution over eager execution for this particular model when +using a single NVIDIA Titan X (Pascal) GPU on a host with an Intel Xeon E5-1650 +CPU @ 3.50GHz and a batch size of 32. + +| Benchmark name | examples/second | +| ------------------------------------ | --------------- | +| eager_cudnn_train_large_gpu_batch_20 | 938 | +| graph_cudnn_train_large_gpu_batch_20 | 971 | +| eager_cudnn_train_small_gpu_batch_20 | 2433 | +| graph_cudnn_train_small_gpu_batch_20 | 2585 | + diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py new file mode 100644 index 0000000000000000000000000000000000000000..30bb3c8ad33d38453bd96a76c7770071e24bb034 --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py @@ -0,0 +1,359 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Penn Treebank RNN model definition compatible with eager execution. + +Model similar to +https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb + +Usage: python ./rnn_ptb.py --data-path= + +Penn Treebank (PTB) dataset from: +http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz +""" +import argparse +import os +import sys +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn +from tensorflow.contrib.eager.python import tfe + + +class RNN(tfe.Network): + """A static RNN. + + Similar to tf.nn.static_rnn, implemented as a tf.layer.Layer. + """ + + def __init__(self, hidden_dim, num_layers, keep_ratio): + super(RNN, self).__init__() + self.keep_ratio = keep_ratio + for _ in range(num_layers): + self.track_layer(tf.nn.rnn_cell.BasicLSTMCell(num_units=hidden_dim)) + + def call(self, input_seq, training): + batch_size = int(input_seq.shape[1]) + for c in self.layers: + state = c.zero_state(batch_size, tf.float32) + outputs = [] + input_seq = tf.unstack(input_seq, num=int(input_seq.shape[0]), axis=0) + for inp in input_seq: + output, state = c(inp, state) + outputs.append(output) + + input_seq = tf.stack(outputs, axis=0) + if training: + input_seq = tf.nn.dropout(input_seq, self.keep_ratio) + return input_seq, None + + +class Embedding(tf.layers.Layer): + """An Embedding layer.""" + + def __init__(self, vocab_size, embedding_dim, **kwargs): + super(Embedding, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.embedding_dim = embedding_dim + + def build(self, _): + self.embedding = self.add_variable( + "embedding_kernel", + shape=[self.vocab_size, self.embedding_dim], + dtype=tf.float32, + initializer=tf.random_uniform_initializer(-0.1, 0.1), + trainable=True) + + def call(self, x): + return tf.nn.embedding_lookup(self.embedding, x) + + +class PTBModel(tfe.Network): + """LSTM for word language modelling. + + Model described in: + (Zaremba, et. al.) Recurrent Neural Network Regularization + http://arxiv.org/abs/1409.2329 + + See also: + https://github.com/tensorflow/models/tree/master/tutorials/rnn/ptb + """ + + def __init__(self, + vocab_size, + embedding_dim, + hidden_dim, + num_layers, + dropout_ratio, + use_cudnn_rnn=True): + super(PTBModel, self).__init__() + + self.keep_ratio = 1 - dropout_ratio + self.use_cudnn_rnn = use_cudnn_rnn + self.embedding = self.track_layer(Embedding(vocab_size, embedding_dim)) + + if self.use_cudnn_rnn: + self.rnn = cudnn_rnn.CudnnLSTM( + num_layers, hidden_dim, dropout=dropout_ratio) + else: + self.rnn = RNN(hidden_dim, num_layers, self.keep_ratio) + self.track_layer(self.rnn) + + self.linear = self.track_layer( + tf.layers.Dense( + vocab_size, + kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))) + self._output_shape = [-1, embedding_dim] + + def call(self, input_seq, training): + """Run the forward pass of PTBModel. + + Args: + input_seq: [length, batch] shape int64 tensor. + training: Is this a training call. + Returns: + outputs tensors of inference. + """ + y = self.embedding(input_seq) + if training: + y = tf.nn.dropout(y, self.keep_ratio) + y, _ = self.rnn(y, training=training) + return self.linear(tf.reshape(y, self._output_shape)) + + +def clip_gradients(grads_and_vars, clip_ratio): + gradients, variables = zip(*grads_and_vars) + clipped, _ = tf.clip_by_global_norm(gradients, clip_ratio) + return zip(clipped, variables) + + +def loss_fn(model, inputs, targets, training): + labels = tf.reshape(targets, [-1]) + outputs = model(inputs, training) + return tf.reduce_mean( + tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=labels, logits=outputs)) + + +def _divide_into_batches(data, batch_size): + """Convert a sequence to a batch of sequences.""" + nbatch = data.shape[0] // batch_size + data = data[:nbatch * batch_size] + data = data.reshape(batch_size, -1).transpose() + return data + + +def _get_batch(data, i, seq_len): + slen = min(seq_len, data.shape[0] - 1 - i) + inputs = data[i:i + slen, :] + target = data[i + 1:i + 1 + slen, :] + return tf.constant(inputs), tf.constant(target) + + +def evaluate(model, data): + """evaluate an epoch.""" + total_loss = 0.0 + total_batches = 0 + start = time.time() + for _, i in enumerate(range(0, data.shape[0] - 1, FLAGS.seq_len)): + inp, target = _get_batch(data, i, FLAGS.seq_len) + loss = loss_fn(model, inp, target, training=False) + total_loss += loss.numpy() + total_batches += 1 + time_in_ms = (time.time() - start) * 1000 + sys.stderr.write("eval loss %.2f (eval took %d ms)\n" % + (total_loss / total_batches, time_in_ms)) + return total_loss + + +def train(model, optimizer, train_data, sequence_length, clip_ratio): + """training an epoch.""" + + def model_loss(inputs, targets): + return loss_fn(model, inputs, targets, training=True) + + grads = tfe.implicit_gradients(model_loss) + + total_time = 0 + for batch, i in enumerate(range(0, train_data.shape[0] - 1, sequence_length)): + train_seq, train_target = _get_batch(train_data, i, sequence_length) + start = time.time() + optimizer.apply_gradients( + clip_gradients(grads(train_seq, train_target), clip_ratio)) + total_time += (time.time() - start) + if batch % 10 == 0: + time_in_ms = (total_time * 1000) / (batch + 1) + sys.stderr.write("batch %d: training loss %.2f, avg step time %d ms\n" % + (batch, model_loss(train_seq, train_target).numpy(), + time_in_ms)) + + +class Datasets(object): + """Processed form of the Penn Treebank dataset.""" + + def __init__(self, path): + """Load the Penn Treebank dataset. + + Args: + path: Path to the data/ directory of the dataset from from Tomas Mikolov's + webpage - http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz + """ + + self.word2idx = {} # string -> integer id + self.idx2word = [] # integer id -> word string + # Files represented as a list of integer ids (as opposed to list of string + # words). + self.train = self.tokenize(os.path.join(path, "ptb.train.txt")) + self.valid = self.tokenize(os.path.join(path, "ptb.valid.txt")) + + def vocab_size(self): + return len(self.idx2word) + + def add(self, word): + if word not in self.word2idx: + self.idx2word.append(word) + self.word2idx[word] = len(self.idx2word) - 1 + + def tokenize(self, path): + """Read text file in path and return a list of integer token ids.""" + tokens = 0 + with tf.gfile.Open(path, "r") as f: + for line in f: + words = line.split() + [""] + tokens += len(words) + for word in words: + self.add(word) + + # Tokenize file content + with tf.gfile.Open(path, "r") as f: + ids = np.zeros(tokens).astype(np.int64) + token = 0 + for line in f: + words = line.split() + [""] + for word in words: + ids[token] = self.word2idx[word] + token += 1 + + return ids + + +def small_model(use_cudnn_rnn): + """Returns a PTBModel with a 'small' configuration.""" + return PTBModel( + vocab_size=10000, + embedding_dim=200, + hidden_dim=200, + num_layers=2, + dropout_ratio=0., + use_cudnn_rnn=use_cudnn_rnn) + + +def large_model(use_cudnn_rnn): + """Returns a PTBModel with a 'large' configuration.""" + return PTBModel( + vocab_size=10000, + embedding_dim=650, + hidden_dim=650, + num_layers=2, + dropout_ratio=0.5, + use_cudnn_rnn=use_cudnn_rnn) + + +def test_model(use_cudnn_rnn): + """Returns a tiny PTBModel for unit tests.""" + return PTBModel( + vocab_size=100, + embedding_dim=20, + hidden_dim=20, + num_layers=2, + dropout_ratio=0., + use_cudnn_rnn=use_cudnn_rnn) + + +def main(_): + tfe.enable_eager_execution() + + if not FLAGS.data_path: + raise ValueError("Must specify --data-path") + corpus = Datasets(FLAGS.data_path) + train_data = _divide_into_batches(corpus.train, FLAGS.batch_size) + eval_data = _divide_into_batches(corpus.valid, 10) + + have_gpu = tfe.num_gpus() > 0 + use_cudnn_rnn = not FLAGS.no_use_cudnn_rnn and have_gpu + + with tfe.restore_variables_on_create( + tf.train.latest_checkpoint(FLAGS.logdir)): + with tf.device("/device:GPU:0" if have_gpu else None): + # Make learning_rate a Variable so it can be included in the checkpoint + # and we can resume training with the last saved learning_rate. + learning_rate = tfe.Variable(20.0, name="learning_rate") + sys.stderr.write("learning_rate=%f\n" % learning_rate.numpy()) + model = PTBModel(corpus.vocab_size(), FLAGS.embedding_dim, + FLAGS.hidden_dim, FLAGS.num_layers, FLAGS.dropout, + use_cudnn_rnn) + optimizer = tf.train.GradientDescentOptimizer(learning_rate) + + best_loss = None + for _ in range(FLAGS.epoch): + train(model, optimizer, train_data, FLAGS.seq_len, FLAGS.clip) + eval_loss = evaluate(model, eval_data) + if not best_loss or eval_loss < best_loss: + if FLAGS.logdir: + tfe.Saver(model.trainable_weights + [learning_rate]).save( + os.path.join(FLAGS.logdir, "ckpt")) + best_loss = eval_loss + else: + learning_rate.assign(learning_rate / 4.0) + sys.stderr.write("eval_loss did not reduce in this epoch, " + "changing learning rate to %f for the next epoch\n" % + learning_rate.numpy()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--data-path", + type=str, + default="", + help="Data directory of the Penn Treebank dataset from " + "http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz") + parser.add_argument( + "--logdir", type=str, default="", help="Directory for checkpoint.") + parser.add_argument( + "--epoch", type=int, default=20, help="Number of epoches.") + parser.add_argument("--batch-size", type=int, default=20, help="Batch size.") + parser.add_argument( + "--seq-len", type=int, default=35, help="Sequence length.") + parser.add_argument( + "--embedding-dim", type=int, default=200, help="Embedding dimension.") + parser.add_argument( + "--hidden-dim", type=int, default=200, help="Hidden layer dimension.") + parser.add_argument( + "--num-layers", type=int, default=2, help="Number of RNN layers.") + parser.add_argument( + "--dropout", type=float, default=0.2, help="Drop out ratio.") + parser.add_argument( + "--clip", type=float, default=0.25, help="Gradient clipping ratio.") + parser.add_argument( + "--no-use-cudnn-rnn", + action="store_true", + default=False, + help="Disable the fast CuDNN RNN (when no gpu)") + + FLAGS, unparsed = parser.parse_known_args() + tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py new file mode 100644 index 0000000000000000000000000000000000000000..63b5c4c54d13e9c2448ec1f572ca1389f2443bef --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_graph_test.py @@ -0,0 +1,164 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for PTBModel used for graph construction.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python.examples.rnn_ptb import rnn_ptb + + +class PTBTest(tf.test.TestCase): + + def testTrain(self): + batch_size = 20 + sequence_length = 35 + with tf.Graph().as_default(), tf.device(tf.test.gpu_device_name()): + inputs_ph = tf.placeholder(tf.int64, [sequence_length, batch_size], + "inputs") + labels_ph = tf.placeholder(tf.int64, [sequence_length, batch_size], + "labels") + + inputs = np.ones(inputs_ph.shape.as_list(), dtype=np.int64) + labels = np.ones(labels_ph.shape.as_list(), dtype=np.int64) + + model = rnn_ptb.test_model(tf.test.is_gpu_available()) + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + loss = rnn_ptb.loss_fn(model, inputs_ph, labels_ph, training=True) + grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) + train_op = optimizer.apply_gradients(grads) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + sess.run(train_op, feed_dict={inputs_ph: inputs, labels_ph: labels}) + sess.run( + [train_op, loss], feed_dict={ + inputs_ph: inputs, + labels_ph: labels + }) + + +class PTBBenchmark(tf.test.Benchmark): + + BATCH_SIZE = 20 + SEQ_LEN = 35 + + def _report(self, label, start, num_iters, device, batch_size): + wall_time = (time.time() - start) / num_iters + dev = "cpu" if "cpu" in device.lower() else "gpu" + name = "%s_%s_batch_%d" % (label, dev, batch_size) + examples_per_sec = batch_size / wall_time + self.report_benchmark( + iters=num_iters, + wall_time=wall_time, + name=name, + extras={ + "examples_per_sec": examples_per_sec + }) + + def _benchmark_apply(self, label, model): + num_iters = 100 + num_warmup = 10 + dataset = tf.data.Dataset.from_tensors( + tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], + dtype=tf.int64)).repeat(num_iters + num_warmup) + inputs = dataset.make_one_shot_iterator().get_next() + + with tf.device(tf.test.gpu_device_name()): + outputs = model(inputs, training=True) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for _ in range(num_warmup): + sess.run(outputs) + gc.collect() + + start = time.time() + for _ in range(num_iters): + sess.run(outputs) + self._report(label, start, num_iters, + tf.test.gpu_device_name(), PTBBenchmark.BATCH_SIZE) + + def benchmark_apply_small(self): + self._benchmark_apply("graph_apply_small", rnn_ptb.small_model(False)) + + def benchmark_apply_large(self): + self._benchmark_apply("graph_apply_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_apply_small(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_apply("graph_cudnn_apply_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_apply_large(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_apply("graph_cudnn_apply_large", rnn_ptb.large_model(True)) + + def _benchmark_train(self, label, model): + num_iters = 100 + num_warmup = 10 + dataset = tf.data.Dataset.from_tensors( + tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], + dtype=tf.int64)).repeat(num_iters + num_warmup) + # inputs and labels have the same shape + dataset = tf.data.Dataset.zip((dataset, dataset)) + (inputs, labels) = dataset.make_one_shot_iterator().get_next() + + with tf.device(tf.test.gpu_device_name()): + optimizer = tf.train.GradientDescentOptimizer(learning_rate=1.0) + loss = rnn_ptb.loss_fn(model, inputs, labels, training=True) + grads = rnn_ptb.clip_gradients(optimizer.compute_gradients(loss), 0.25) + train_op = optimizer.apply_gradients(grads) + + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + for _ in range(num_warmup): + sess.run(train_op) + gc.collect() + start = time.time() + for _ in range(num_iters): + sess.run(train_op) + self._report(label, start, num_iters, + tf.test.gpu_device_name(), PTBBenchmark.BATCH_SIZE) + + def benchmark_train_small(self): + self._benchmark_train("graph_train_small", rnn_ptb.small_model(False)) + + def benchmark_train_large(self): + self._benchmark_train("graph_train_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_train_small(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_train("graph_cudnn_train_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_train_large(self): + if not tf.test.is_gpu_available(): + return + self._benchmark_train("graph_cudnn_train_large", rnn_ptb.large_model(True)) + + +if __name__ == "__main__": + tf.test.main() diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py new file mode 100644 index 0000000000000000000000000000000000000000..b279bc4a7c3510b6a59bc618b531141beebdfaab --- /dev/null +++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb_test.py @@ -0,0 +1,154 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for PTBModel with eager execution enabled.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import gc +import time + +import numpy as np +import tensorflow as tf + +from tensorflow.contrib.eager.python import tfe +from tensorflow.contrib.eager.python.examples.rnn_ptb import rnn_ptb + + +def device(): + return "/device:GPU:0" if tfe.num_gpus() else "/device:CPU:0" + + +class PTBTest(tf.test.TestCase): + + def testTrain(self): + model = rnn_ptb.test_model(tfe.num_gpus() > 0) + sequence_length = 35 + data = np.ones([4 * sequence_length, 20], dtype=np.int64) + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(1.0) + # Train two epochs + rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) + rnn_ptb.train(model, optimizer, data, sequence_length, 0.25) + + def testApply(self): + model = rnn_ptb.test_model(tfe.num_gpus() > 0) + with tf.device(device()): + model(tf.ones([35, 20], dtype=tf.int64), training=False) + + +def force_gpu_sync(): + if tfe.num_gpus(): + tf.constant(1).gpu().cpu() + + +class PTBBenchmark(tf.test.Benchmark): + + BATCH_SIZE = 20 + SEQ_LEN = 35 + + def _report(self, label, start, num_iters, dev, batch_size): + wall_time = (time.time() - start) / num_iters + dev = "cpu" if "cpu" in dev.lower() else "gpu" + name = "%s_%s_batch_%d" % (label, dev, batch_size) + examples_per_sec = batch_size / wall_time + self.report_benchmark( + iters=num_iters, + wall_time=wall_time, + name=name, + extras={ + "examples_per_sec": examples_per_sec + }) + + def _benchmark_apply(self, label, model): + with tf.device(device()): + sequence_batch = tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64) + + for _ in range(10): # Warmup + model(sequence_batch, training=False).cpu() + gc.collect() + + start = time.time() + iters = 100 + for _ in range(iters): + model(sequence_batch, training=False).cpu() + self._report(label, start, iters, device(), int(sequence_batch.shape[1])) + + def benchmark_apply_small(self): + self._benchmark_apply("eager_apply_small", rnn_ptb.small_model(False)) + + def benchmark_apply_large(self): + self._benchmark_apply("eager_apply_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_apply_small(self): + if not tfe.num_gpus(): + return + self._benchmark_apply("eager_cudnn_apply_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_apply_large(self): + if not tfe.num_gpus(): + return + self._benchmark_apply("eager_cudnn_apply_large", rnn_ptb.large_model(True)) + + def _benchmark_train(self, label, model): + with tf.device(device()): + optimizer = tf.train.GradientDescentOptimizer(1.) + + def model_loss(inputs, targets): + return rnn_ptb.loss_fn(model, inputs, targets, training=True) + + grads = tfe.implicit_gradients(model_loss) + + sequence_batch = tf.ones( + [PTBBenchmark.SEQ_LEN, PTBBenchmark.BATCH_SIZE], dtype=tf.int64) + + def step(): + optimizer.apply_gradients( + rnn_ptb.clip_gradients(grads(sequence_batch, sequence_batch), 0.25)) + + for _ in range(10): # Warmup + step() + force_gpu_sync() + gc.collect() + + start = time.time() + iters = 100 + for _ in range(iters): + step() + force_gpu_sync() + self._report(label, start, iters, device(), int(sequence_batch.shape[1])) + + def benchmark_train_small(self): + self._benchmark_train("eager_train_small", rnn_ptb.small_model(False)) + + def benchmark_train_large(self): + self._benchmark_train("eager_train_large", rnn_ptb.large_model(False)) + + def benchmark_cudnn_train_small(self): + if not tfe.num_gpus(): + return + self._benchmark_train("eager_cudnn_train_small", rnn_ptb.small_model(True)) + + def benchmark_cudnn_train_large(self): + if not tfe.num_gpus(): + return + self._benchmark_train("eager_cudnn_train_large", rnn_ptb.large_model(True)) + + +if __name__ == "__main__": + tfe.enable_eager_execution() + tf.test.main() diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md new file mode 100644 index 0000000000000000000000000000000000000000..e76745a807cb10adf2aedc56e69cea0ceded3ad7 --- /dev/null +++ b/tensorflow/contrib/eager/python/g3doc/guide.md @@ -0,0 +1,899 @@ +# TensorFlow Eager Execution + +## What is this? + +Eager execution is a feature that makes TensorFlow execute operations +immediately: concrete values are returned, instead of a computational graph to +be executed later. + +As a result, enabling eager execution provides: + +- A [NumPy](http://www.numpy.org/)-like library for numerical computation with + support for GPU acceleration and automatic differentiation. +- A flexible platform for machine learning research and experimentation. + +Eager execution is under active development. This guide walks through an +alpha/preview release. In particular, not all TensorFlow APIs currently work +with eager execution enabled, and some models may be slow to execute, compared +to models defined without using eager execution. + +## Installation + +Eager execution is **not** included in the latest release (version 1.4) of +TensorFlow. To use it, you will need to [build TensorFlow from +source](https://www.tensorflow.org/install/install_sources) or install the +nightly builds. + +For example, the nightly builds can be installed using `pip`: + +- `pip install tf-nightly` (for CPU-only TensorFlow) +- `pip install tf-nightly-gpu` (for GPU-enabled TensorFlow) + +Or using `docker`, with [Jupyter Notebook](http://jupyter.org/) support: + +```sh +# For CPU-only TensorFlow +docker pull tensorflow/tensorflow:nightly +docker run -it -p 8888:8888 tensorflow/tensorflow:nightly + +# For GPU-enabled TensorFlow: +# (Requires https://github.com/NVIDIA/nvidia-docker) +nvidia-docker pull tensorflow/tensorflow:nightly-gpu +nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu +``` + +## Getting Started + +With TensorFlow installed, eager execution is enabled via a single call: + +```python +import tensorflow as tf + +import tensorflow.contrib.eager as tfe + +tfe.enable_eager_execution() +``` + +Enabling eager execution changes how TensorFlow functions behave (in particular, +`Tensor` objects will reference concrete values instead of being symbolic +handles to nodes in a computational graph). As a result, eager execution should +be enabled at the beginning of a program and cannot be disabled afterwards in +the same program. + +Code examples in the rest of this guide assume that eager execution has been +enabled. + +## A library for numerical computation + +A significant fraction of the [TensorFlow +API](https://www.tensorflow.org/api_docs/python/) consists of numerical +operations: +[arithmetic operations](https://www.tensorflow.org/api_guides/python/math_ops#Arithmetic_Operators), +[matrix operations](https://www.tensorflow.org/api_guides/python/math_ops#Matrix_Math_Functions), +[linear algebra operations](https://www.tensorflow.org/versions/master/api_docs/python/tf/linalg), +etc. + +With eager execution enabled, these operations consume and return +multi-dimensional arrays as `Tensor` objects, similar to NumPy +[`ndarray`s](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ndarray.html). +For example: + +```python +# Multiply two 2x2 matrices +x = tf.matmul([[1, 2], + [3, 4]], + [[4, 5], + [6, 7]]) +# Add one to each element +# (tf.add supports broadcasting) +y = tf.add(x, 1) + +# Create a random random 5x3 matrix +z = tf.random_uniform([5, 3]) + +print(x) +print(y) +print(z) +``` + +Output: + +``` +tf.Tensor( +[[16 19] + [36 43]], shape=(2, 2), dtype=int32) +tf.Tensor( +[[17 20] + [37 44]], shape=(2, 2), dtype=int32) +tf.Tensor( +[[ 0.25058532 0.0929395 0.54113817] + [ 0.3108716 0.93350542 0.84909797] + [ 0.53081679 0.12788558 0.01767385] + [ 0.29725885 0.33540785 0.83588314] + [ 0.38877153 0.39720535 0.78914213]], shape=(5, 3), dtype=float32) +``` + +For convenience, these operations can also be triggered via operator overloading +of the `Tensor` object. For example, the `+` operator is equivalent to `tf.add`, +`-` to `tf.subtract`, `*` to `tf.multiply`, etc.: + +```python +x = (tf.ones([1], dtype=tf.float32) + 1) * 2 - 1 +print(x) +``` + +Output: + +``` +tf.Tensor([ 3.], shape=(1,), dtype=float32) +``` + +### Converting to and from NumPy + +The operations above automatically convert Python objects (like lists of +numbers) and NumPy arrays to `Tensor` objects. `Tensor` objects can also be used +as NumPy arrays by numpy operations. + +```python +import numpy as np + +x = tf.add(1, 1) # tf.Tensor with a value of 2 +y = tf.add(np.array(1), np.array(1)) # tf.Tensor with a value of 2 +z = np.multiply(x, y) # numpy.int64 with a value of 4 +``` + +Alternatively, they can be explicitly converted using +[`tf.constant`](https://www.tensorflow.org/api_docs/python/tf/constant), as +shown in the next example. + +Conversely, you can call the `numpy()` method of a `Tensor` object' to obtain +its NumPy `ndarray` value. For example: + +```python +import numpy as np + +np_x = np.array(2., dtype=np.float32) +x = tf.constant(np_x) + +py_y = 3. +y = tf.constant(py_y) + +z = x + y + 1 + +print(z) +print(z.numpy()) +``` + +Output: + +``` +tf.Tensor(6.0, shape=(), dtype=float32) +6.0 +``` + +### GPU acceleration + +Many TensorFlow operations support GPU acceleration. With eager execution +enabled, [computation is *not* automatically +offloaded](https://www.tensorflow.org/tutorials/using_gpu) to GPUs. Instead, you +must explicitly specify when GPUs should be used. + +The simplest way to do this is to enclose your computation in a `with +tf.device('/gpu:0')` block. Also of interest is the `tfe.num_gpus()` function, +which returns the number of available GPUs. + +For example, consider this snippet to measure the time to multiply two 1000x1000 +matrices on CPU: + +```python +import time + +def measure(x): + # The very first time a GPU is used by TensorFlow, it is initialized. + # So exclude the first run from timing. + tf.matmul(x, x) + + start = time.time() + for i in range(10): + tf.matmul(x, x) + end = time.time() + + return "Took %s seconds to multiply a %s matrix by itself 10 times" % (end - start, x.shape) + +# Run on CPU: +with tf.device("/cpu:0"): + print("CPU: %s" % measure(tf.random_normal([1000, 1000]))) + +# If a GPU is available, run on GPU: +if tfe.num_gpus() > 0: + with tf.device("/gpu:0"): + print("GPU: %s" % measure(tf.random_normal([1000, 1000]))) +``` + +Output (exact numbers will depend on the characteristics of the hardware): + +```python +CPU: Took 0.145531892776 seconds to multiply a (1000, 1000) matrix by itself 10 times +GPU: Took 0.000458955764771 seconds to multiply a (1000, 1000) matrix by itself 10 times +``` + +Alternatively, methods on the `Tensor` object can be used to explicitly copy the +`Tensor` to a different device. Operations are typically executed on the device +on which the inputs are placed. For example: + +```python +x = tf.random_normal([10, 10]) + +x_gpu0 = x.gpu() +x_cpu = x.cpu() + +_ = tf.matmul(x_cpu, x_cpu) # Runs on CPU +_ = tf.matmul(x_gpu0, x_gpu0) # Runs on GPU:0 + +if tfe.num_gpus() > 1: + x_gpu1 = x.gpu(1) + _ = tf.matmul(x_gpu1, x_gpu1) # Runs on GPU:1 +``` + +### Automatic Differentiation + +[Automatic +differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) is +very useful when implementing many machine learning algorithms (e.g., +[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training +neural networks). For this purpose, TensorFlow eager execution provides an +[autograd](https://github.com/HIPS/autograd)-style API for automatic +differentiation. Specifically, the functions: + +- `tfe.gradients_function(f)`: Returns a Python function that computes the + derivatives of the Python function `f` with respect to its arguments. `f` + must return a scalar value. When the returned function is invoked, it + returns a list of `Tensor` objects (one element for each argument of `f`). +- `tfe.value_and_gradients_function(f)`: Similar to `tfe.gradients_function`, + except that when the returned function is invoked, it returns the value of + `f` in addition to the list of derivatives of `f` with respect to its + arguments. + +These functions naturally apply to higher order differentiation as well. For +example: + +```python +def f(x): + return tf.multiply(x, x) # Or x * x +assert 9 == f(3.).numpy() + +df = tfe.gradients_function(f) +assert 6 == df(3.)[0].numpy() + +# Second order deriviative. +d2f = tfe.gradients_function(lambda x: df(x)[0]) +assert 2 == d2f(3.)[0].numpy() + +# Third order derivative. +d3f = tfe.gradients_function(lambda x : d2f(x)[0]) +assert 0 == d3f(3.)[0].numpy() +``` + +These functions can be used to train models. For example, consider the following +simple linear regression model: + +```python +def prediction(input, weight, bias): + return input * weight + bias + +# A toy dataset of points around 3 * x + 2 +NUM_EXAMPLES = 1000 +training_inputs = tf.random_normal([NUM_EXAMPLES]) +noise = tf.random_normal([NUM_EXAMPLES]) +training_outputs = training_inputs * 3 + 2 + noise + +# A loss function: Mean-squared error +def loss(weight, bias): + error = prediction(training_inputs, weight, bias) - training_outputs + return tf.reduce_mean(tf.square(error)) + +# Function that returns the the derivative of loss with respect to +# weight and bias +grad = tfe.gradients_function(loss) + +# Train for 200 steps (starting from some random choice for W and B, on the same +# batch of data). +W = 5. +B = 10. +learning_rate = 0.01 +print("Initial loss: %f" % loss(W, B).numpy()) +for i in range(200): + (dW, dB) = grad(W, B) + W -= dW * learning_rate + B -= dB * learning_rate + if i % 20 == 0: + print("Loss at step %d: %f" % (i, loss(W, B).numpy())) +print("Final loss: %f" % loss(W, B).numpy()) +print("W, B = %f, %f" % (W.numpy(), B.numpy())) +``` + +Output: (the exact numbers may vary depending on the randomness in noise) + +``` +Initial loss: 66.730003 +Loss at step 0: 64.200096 +Loss at step 20: 29.872814 +Loss at step 40: 14.233772 +Loss at step 60: 7.090570 +Loss at step 80: 3.819887 +Loss at step 100: 2.318821 +Loss at step 120: 1.628385 +Loss at step 140: 1.310142 +Loss at step 160: 1.163167 +Loss at step 180: 1.095162 +Final loss: 1.064711 +W, B = 3.094944, 2.161383 +``` + +To utilize the GPU, place the code above within a `with tf.device("/gpu:0"):` +block. (However, this particular model, with only two floating point parameters, +is unlikely to benefit from GPU acceleration.) + +### Customizing gradients + +One may want to define custom gradients for an operation, or for a function. +This may be useful for multiple reasons, including providing a more efficient +or more [numerically stable](https://en.wikipedia.org/wiki/Numerical_stability) +gradient for a sequence of operations. + +For example, consider the function `log(1 + e^x)`, which commonly occurs in the +computation of cross entropy and log likelihoods. + +```python +def log1pexp(x): +  return tf.log(1 + tf.exp(x)) +grad_log1pexp = tfe.gradients_function(log1pexp) + +# Works fine at x = 0. +assert 0.5 == float(grad_log1pexp(0.)[0]) + +# Returns a `nan` at x = 100 due to numerical instability. +import math +assert math.isnan(float(grad_log1pexp(100.)[0])) +``` + +We can define a custom gradient for the above function that analytically +simplifies the gradient expression. + +```python +@tfe.custom_gradient +def log1pexp(x): +  e = tf.exp(x) +  def grad(dy): +    return dy * (1 - 1 / (1 + e)) +  return tf.log(1 + e), grad +grad_log1pexp = tfe.gradients_function(log1pexp) + +# Works as before at x = 0. +assert 0.5 == float(grad_log1pexp(0.)[0]) + +# But now works at x = 100 as well. +assert 1.0 == float(grad_log1pexp(100.)[0]) +``` +Also notice how the gradient function implementation reuses an expression +(`tf.exp(x)`) computed during the forward pass, hence making the gradient +computation more efficient by avoiding redundant computation. + +## Building and training models + +In practice, your computation may have many parameters to be optimized (by +computing derivatives). Encapsulating them into re-usable classes/objects +makes the code easier to follow than writing a single top-level function with +many arguments. + +In fact, eager execution encourages use of the [Keras](https://keras.io)-style +"Layer" classes in the +[`tf.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/layers) +module. + +Furthermore, you may want to apply more sophisticated techniques to compute +parameter updates, such as those in +[`tf.train.Optimizer`](https://www.tensorflow.org/api_guides/python/train#Optimizers) +implementations. + +This next section walks through using the same `Optimizer` and `Layer` APIs used +to build trainable TensorFlow graphs in an environment where eager execution is +enabled. + +### Variables and Optimizers + +`tfe.Variable` objects store mutable `Tensor` values that can be accessed during +training, making automatic differentiation easier. In particular, parameters of +a model can be encapsulated in Python classes as variables. + +`tfe.gradients_function(f)` introduced earlier computes the derivatives of `f` +with respect to its arguments. However, it requires all parameters of interest +to be arguments of `f`, which becomes cumbersome when `f` depends on a large +number of trainable parameters. + +`tfe.implicit_gradients` is an alternative function with some useful properties: + +- It computes the derivatives of `f` with respect to all the `tfe.Variable`s + used by `f`. +- When the returned function is invoked, it returns a list of + (gradient value, Variable object) tuples. + +Representing model parameters as `Variable` objects, along with the use of +`tfe.implicit_gradients`, typically results in better encapsulation. For +example, the linear regression model described above can be written into a +class: + +```python +class Model(object): + def __init__(self): + self.W = tfe.Variable(5., name='weight') + self.B = tfe.Variable(10., name='bias') + + def predict(self, inputs): + return inputs * self.W + self.B + + +# The loss function to be optimized +def loss(model, inputs, targets): + error = model.predict(inputs) - targets + return tf.reduce_mean(tf.square(error)) + +# A toy dataset of points around 3 * x + 2 +NUM_EXAMPLES = 1000 +training_inputs = tf.random_normal([NUM_EXAMPLES]) +noise = tf.random_normal([NUM_EXAMPLES]) +training_outputs = training_inputs * 3 + 2 + noise + +# Define: +# 1. A model +# 2. Derivatives of a loss function with respect to model parameters +# 3. A strategy for updating the variables based on the derivatives +model = Model() +grad = tfe.implicit_gradients(loss) +optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01) + +# The training loop +print("Initial loss: %f" % + loss(model, training_inputs, training_outputs).numpy()) +for i in range(201): + optimizer.apply_gradients(grad(model, training_inputs, training_outputs)) + if i % 20 == 0: + print("Loss at step %d: %f" % + (i, loss(model, training_inputs, training_outputs).numpy())) +print("Final loss: %f" % loss(model, training_inputs, training_outputs).numpy()) +print("W, B = %s, %s" % (model.W.numpy(), model.B.numpy())) +``` + +Output: + +``` +Initial loss: 69.693184 +Loss at step 0: 66.987854 +Loss at step 20: 30.553387 +Loss at step 40: 14.250237 +Loss at step 60: 6.955020 +Loss at step 80: 3.690550 +Loss at step 100: 2.229739 +Loss at step 120: 1.576032 +Loss at step 140: 1.283496 +Loss at step 160: 1.152584 +Loss at step 180: 1.093999 +Final loss: 1.067780 +W, B = 3.0114281, 2.0865183 +``` + +Using `implicit_gradients` avoids the need to provide all the trainable +parameters of the model as arguments to the `loss` function. + +### Using Keras and the Layers API + +[Keras](https://keras.io) is a popular API for defining model structures. The +[`tf.keras.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/layers) +module provides a set of building blocks for models and is implemented using the +`tf.layers.Layer` subclasses in the +[`tf.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/layers) +module. We encourage the use of these same building blocks when using +TensorFlow's eager execution feature. For example, the very same linear +regression model can be built using `tf.layers.Dense`: + +```python +class Model(object): + def __init__(self): + self.layer = tf.layers.Dense(1) + + def predict(self, inputs): + return self.layer(inputs) +``` + +The `tf.layers` API makes it more convenient to define more sophisticated +models. For example, the following will train an MNIST model: + +```python +class MNISTModel(object): + def __init__(self, data_format): + # 'channels_first' is typically faster on GPUs + # while 'channels_last' is typically faster on CPUs. + # See: https://www.tensorflow.org/performance/performance_guide#data_formats + if data_format == 'channels_first': + self._input_shape = [-1, 1, 28, 28] + else: + self._input_shape = [-1, 28, 28, 1] + self.conv1 = tf.layers.Conv2D(32, 5, + padding='same', + activation=tf.nn.relu, + data_format=data_format) + self.max_pool2d = tf.layers.MaxPooling2D( + (2, 2), (2, 2), padding='same', data_format=data_format) + self.conv2 = tf.layers.Conv2D(64, 5, + padding='same', + activation=tf.nn.relu, + data_format=data_format) + self.dense1 = tf.layers.Dense(1024, activation=tf.nn.relu) + self.dropout = tf.layers.Dropout(0.5) + self.dense2 = tf.layers.Dense(10) + + def predict(self, inputs): + x = tf.reshape(inputs, self._input_shape) + x = self.max_pool2d(self.conv1(x)) + x = self.max_pool2d(self.conv2(x)) + x = tf.layers.flatten(x) + x = self.dropout(self.dense1(x)) + return self.dense2(x) + +def loss(model, inputs, targets): + return tf.reduce_mean( + tf.nn.softmax_cross_entropy_with_logits( + logits=model.predict(inputs), labels=targets)) + + +# Load the training and validation data +from tensorflow.examples.tutorials.mnist import input_data +data = input_data.read_data_sets("./mnist_data", one_hot=True) + +# Train +device = "gpu:0" if tfe.num_gpus() else "cpu:0" +model = MNISTModel('channels_first' if tfe.num_gpus() else 'channels_last') +optimizer = tf.train.AdamOptimizer(learning_rate=1e-4) +grad = tfe.implicit_gradients(loss) +for i in range(20001): + with tf.device(device): + (inputs, targets) = data.train.next_batch(50) + optimizer.apply_gradients(grad(model, inputs, targets)) + if i % 100 == 0: + print("Step %d: Loss on training set : %f" % + (i, loss(model, inputs, targets).numpy())) +print("Loss on test set: %f" % loss(model, data.test.images, data.test.labels).numpy()) +``` + +For a more complete example, see +[`tensorflow/contrib/eager/python/examples/mnist.py`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist/mnist.py) + +### Checkpointing trained variables + +TensorFlow Variables (`tfe.Variable`) provides a way to represent shared, +persistent state of your model. The `tfe.Saver` class (which is a thin wrapper +over the +[`tf.train.Saver`](https://www.tensorflow.org/api_docs/python/tf/train/Saver) +class) provides a means to save and restore variables to and from _checkpoints_. + +For example: + +```python +# Create variables. +x = tfe.Variable(10., name='x') +y = tfe.Variable(5., name='y') + +# Create a Saver. +saver = tfe.Saver([x, y]) + +# Assign new values to the variables and save. +x.assign(2.) +saver.save('/tmp/ckpt') + +# Change the variable after saving. +x.assign(11.) +assert 16. == (x + y).numpy() # 11 + 5 + +# Restore the values in the checkpoint. +saver.restore('/tmp/ckpt') + +assert 7. == (x + y).numpy() # 2 + 5 +``` + +### `tfe.Network` + +You may often want to organize your models using classes, like the `MNISTModel` +class described above. We recommend inheriting from the `tfe.Network` class as +it provides conveniences like keeping track of all model variables and methods +to save and restore from checkpoints. + +Sub-classes of `tfe.Network` may register `Layer`s (like classes in +[`tf.layers`](https://www.tensorflow.org/versions/master/api_docs/python/tf/layers), +or [Keras +layers](https://www.tensorflow.org/versions/master/api_docs/python/tf/keras/layers)) +using a call to `self.track_layer()` and define the computation in an +implementation of `call()`. + +Note that `tf.layers.Layer` objects (like `tf.layers.Dense`) create variables +lazily, when the first input is encountered. + +For example, consider the following two-layer neural network: + +```python +class TwoLayerNet(tfe.Network): + def __init__(self): + super(TwoLayerNet, self).__init__() + self.layer1 = self.track_layer( + tf.layers.Dense(2, activation=tf.nn.relu, use_bias=False)) + self.layer2 = self.track_layer(tf.layers.Dense(3, use_bias=False)) + + def call(self, x): + return self.layer2(self.layer1(x)) + +net = TwoLayerNet() + +# No variables created yet +assert 0 == len(net.variables) + +# They are created on first input: +inp = tf.constant([[1.]]) + +# Since input is a 1x1 matrix, net.l1 has 2 units and net.l2 has 3 units, +# the output is the product of a 1x1 matrix with a 1x2 matrix with a 2x3 +# matrix. +assert [1, 3] == net(inp).shape.as_list() # Invoke net; get output shape. +assert 1 == len(net.layer1.variables) +assert 1 == len(net.layer2.variables) +assert 2 == len(net.variables) # weights for each layer. +assert [1, 2] == net.variables[0].shape.as_list() # weights of layer1. +assert [2, 3] == net.variables[1].shape.as_list() # weights of layer2. +``` + +The `tfe.Network` class is itself a sub-class of `tf.layers.Layer`. This allows +instances of `tfe.Network` to be embedded in other networks. For example: + +```python +class ThreeLayerNet(tfe.Network): + def __init__(self): + super(ThreeLayerNet, self).__init__() + self.a = self.track_layer(TwoLayerNet()) + self.b = self.track_layer(tf.layers.Dense(4, use_bias=False)) + + def call(self, x): + return self.b(self.a(x)) + +net = ThreeLayerNet() + +assert [1, 4] == net(inp).shape.as_list() +assert 3 == len(net.variables) +assert [1, 2] == net.variables[0].shape.as_list() +assert [2, 3] == net.variables[1].shape.as_list() +assert [3, 4] == net.variables[2].shape.as_list() +``` + +See more examples in +[`tensorflow/contrib/eager/python/examples`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples). + +`tfe.Saver` in combination with `tfe.restore_variables_on_create` provides a +convenient way to save and load checkpoints without changing the program once +the checkpoint has been created. For example, we can set an objective for the +output of our network, choose an optimizer, and a location for the checkpoint: + +```python +objective = tf.constant([[2., 3., 4., 5.]]) +optimizer = tf.train.AdamOptimizer(0.01) +checkpoint_directory = '/tmp/tfe_example' +checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') +net = ThreeLayerNet() +``` + +Note that variables have not been created yet. We want them to be restored from +a checkpoint, if one exists, so we create them inside a +`tfe.restore_variables_on_create` context manager. Then our training loop is the +same whether starting training or resuming from a previous checkpoint: + +```python +with tfe.restore_variables_on_create( + tf.train.latest_checkpoint(checkpoint_directory)): + global_step = tf.train.get_or_create_global_step() + for _ in range(100): + loss_fn = lambda: tf.norm(net(inp) - objective) + optimizer.minimize(loss_fn, global_step=global_step) + if tf.equal(global_step % 20, 0): + print("Step %d, output %s" % (global_step.numpy(), + net(inp).numpy())) + all_variables = ( + net.variables + + tfe.get_optimizer_variables(optimizer) + + [global_step]) + # Save the checkpoint. + tfe.Saver(all_variables).save(checkpoint_prefix, global_step=global_step) +``` + +The first time it runs, `Network` variables are initialized randomly. Then the +output is trained to match the objective we've set: + +``` +Step 20, output [[ 0.03575622 0.29863232 0.03474367 0.24735749]] +Step 40, output [[ 0.40646029 0.9856872 0.46851286 0.95358551]] +Step 60, output [[ 1.74541104 2.800704 1.79055595 2.74783421]] +Step 80, output [[ 2.14977384 3.44340849 3.96120024 5.16242075]] +Step 100, output [[ 1.99943113 3.02364397 3.93500996 4.9610076 ]] +``` + +In subsequent iterations, variables are initialized with the values read from +the latest checkpoint. Running the same code again, we continue from where we +left off: + +``` +Step 120, output [[ 1.99234128 3.0271616 3.98732996 4.96401167]] +Step 140, output [[ 2.00133467 3.01270437 4.00616646 5.00406504]] +Step 160, output [[ 1.99647415 2.9956708 3.99064088 4.99632359]] +Step 180, output [[ 2.00699997 3.00904822 4.00706148 5.01193142]] +Step 200, output [[ 1.98334622 2.98249531 3.97375059 4.97123432]] +``` + + +### Summaries, metrics and TensorBoard + +[TensorBoard](https://www.tensorflow.org/get_started/summaries_and_tensorboard) +is a popular tool for understanding, debugging and optimizing the model training +process. To benefit from the visualizations offered by TensorBoard, summary +events need to be written during the course of execution of your program. You +might find many Tensorflow programs that include the +[`tf.summary`](https://www.tensorflow.org/api_guides/python/summary) operations +during graph construction. + +`tf.summary` operations are *not* compatible with eager execution, but an +equivalent alternative exists in +[`tf.contrib.summary`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/summary) +that is compatible with both eager execution and graph construction. + +During model construction simply insert summary operations like +`tf.contrib.summary.scalar`. These operations do nothing by default, unless a +summary writer is currently active and a writing policy is set. + +For example, to record summaries once every 100 global steps, use: + +```python +tf.train.get_or_create_global_step() # Ensuring the global step variable exists +writer = tf.contrib.summary.create_summary_file_writer(logdir) + +for _ in range(iterations): + with writer.as_default(): + with tf.contrib.summary.record_summaries_every_n_global_steps(100): + # your model code goes here + tf.contrib.summary.scalar('loss', loss) + # ... +``` + +See the full mnist example in +[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist) +for a full model using `tf.contrib.summary`. + +Similarly to summaries, the metrics in `tf.metrics` are currently not compatible +with eager execution. We instead provide object-oriented metrics in the +`tfe.metrics` package, which are compatible with graph construction as well. + +Metrics in the `tfe.metrics`, such as `tfe.metrics.Mean` and +`tfe.Metrics.Accuracy`, all implement an intuitive object-oriented +interface. Here's an example of how to use the `tfe.metrics.Mean` metric: + +```python +# Metrics are objects, which can be created and destroyed. +my_mean = tfe.metrics.Mean(name='my_mean') +# While a metric is active, you can call it as a function to accumulate into its +# internal state. +my_mean(0.0) +my_mean(10.0) +# Once you've finished updating the metric, you can get its result. In this case +# a simple average over all the calls to it. If a summary writer is active the +# metric will write the appropriate summaries using the metric name. +assert 5.0 == my_mean.result().numpy() +``` + +For a full example of a model using metrics for evaluation, see the mnist +example in +[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist). + +### Input Pipelines + +The discussion above has been centered around the computation executed by your +model. The +[`tf.data`](https://www.tensorflow.org/versions/master/api_docs/python/tf/data) +module provides APIs to build complex input pipelines from simple, reusable +pieces. + +If you're familiar with constructing `tf.data.Dataset` objects when building +TensorFlow graphs, the same API calls are used when eager execution is enabled. +However, the process of iterating over elements of the dataset differs between +eager execution and graph construction. When eager execution is enabled, the +discussion on iterator creation using `make_one_shot_iterator()` and +`get_next()` in the +[Programmer's +Guide](https://www.tensorflow.org/versions/master/programmers_guide/datasets) is +*not* applicable. Instead, a more Pythonic `Iterator` class is available. + +For example: + +```python +# Create a source Dataset from in-memory numpy arrays. +# For reading from files on disk, you may want to use other Dataset classes +# like the TextLineDataset or the TFRecordDataset. +dataset = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]) + +# Apply transformations, shuffling, batching etc. +dataset = dataset.map(tf.square).shuffle(2).batch(2) + +# Use tfe.Iterator to iterate over the dataset. +for x in tfe.Iterator(dataset): + print(x) +``` + +Output: + +``` +tf.Tensor([4 9], shape=(2,), dtype=int32) +tf.Tensor([16 25], shape=(2,), dtype=int32) +tf.Tensor([36 1], shape=(2,), dtype=int32) +``` + +## Interoperating with Graphs + +Eager execution improves the process of model development in Python; however, +because it is in its earliest stages, it does not yet support some features +available to [TensorFlow +graphs](https://www.tensorflow.org/get_started/get_started#the_computational_graph) +that are desirable when deploying models in production. In particular, eager +execution does not yet support distributed training, exporting models (to other +[programming languages](https://www.tensorflow.org/api_docs/), [TensorFlow +serving](https://www.tensorflow.org/serving/), and mobile applications), and +various memory and computation optimizations that are applied to TensorFlow's +dataflow graphs. + +That said, the APIs used to build modes are exactly the same whether executing +eagerly or constructing graphs. This means that you can iteratively develop your +model with eager execution enabled and later, if needed, use the same code to +reap the benefits of representing models as computational graphs. + +For example, +[`mnist.py`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist/mnist.py) +defines a model that is eagerly executed. That same code is used to construct +and execute a graph in +[`mnist_graph_test.py`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist/mnist_graph_test.py). + +Other models in the [examples +directory](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/) +demonstrate this as well. + +Some differences worth noting: + +- There is no notion of a `tf.placeholder` or a `tf.Session` when eager + execution is enabled. +- Many properties on the `tf.Tensor` object, like `tf.Tensor.name`, + `tf.Tensor.op`, `tf.Tensor.inputs` are not meaningful when eager execution + is enabled and their use will raise an `AttributeError`. +- To use `tfe.implicit_gradients` in graph construction, variables must be + created with [`use_resource=True`] provided to + [`tf.get_variable()`](https://www.tensorflow.org/api_docs/python/tf/get_variable) + or + [`tf.variable_scope()`](https://www.tensorflow.org/api_docs/python/tf/variable_scope). +- Some API calls (such as the functional-style `tf.layers.dense`, + `tf.layers.conv2d`) are not compatible with eager execution. Use of such + methods should raise an error indicating the alternative (e.g., the + `tf.layers.Dense` and `tf.layers.Conv2D` classes). + +## What next? + +Please give eager execution a spin. This feature is in early stages and is +evolving, so we welcome your feedback via issues on GitHub (see [known +issues](https://github.com/tensorflow/tensorflow/labels/comp:eager)). + +You may want to browse through some sample code, including benchmarks for some: + +- [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression) +- [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist) +- [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50) +- [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot) +- [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb) + diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py index 63a0f8d9a45dfb12fd1d61a1156b9acf20cf4c81..2ba653af4a2465a17a17ff4ff019e69476f6434e 100644 --- a/tensorflow/contrib/eager/python/metrics_impl.py +++ b/tensorflow/contrib/eager/python/metrics_impl.py @@ -18,62 +18,106 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import re + +from tensorflow.contrib.summary import summary_ops +from tensorflow.python.eager import context +from tensorflow.python.eager import function from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variable_scope +_to_replace = re.compile("[^A-Za-z0-9.]") + + class Metric(object): """A metric holds state for aggregating statistics over an evaluation run. - Users will use Evaluator.add_metric() to add Metric objects to their - evaluation, call them in each step, and then use - Evaluator.all_metric_results() at the end. + Example use with eager execution: + + ```python + m = SomeMetric(...) + for input in ...: + m(input) + print(m.result()) + ``` + + Example use with graph execution: + + ```python + m = SomeMetric(...) + m_placeholder = tf.placeholder(...) + m_update = m(m_placeholder) + # Variables defined in first call, so get the initialization op afterwards. + m_init = m.init_variables() # or tf.global_variables_initializer() + m_result = m.result() + with tf.Session() as sess: + sess.run(m_init) + for input in ...: + sess.run(m_update, feed_dict={m_placeholder: input}) + print(sess.run(m_result)) + ``` Descendants will implement: - * call(): Should follow this pattern: - if not self.built: - self.var = self.add_variable(...) - self.add_update(self.var.assign_add(...)) - * aggregate(): Adds in the state from a list of metrics of the same type - as `self`. (Default of summing all the variables will be fine for most - descendants.) - * result(): Computes and returns a final value for the metric + * `build()`: All variables should be created in this method, by calling + `self.add_variable()` as in: `self.var = self.add_variable(...)` + build() will be called in the first invocation of `__call__()`, with + the same arguments passed `call()`. + * `call()`: Has all updates to variables, as in: + self.var.assign_add(...) + * `result()`: Computes and returns a final value for the metric from the variables in `self`. + + Decendants may override `aggregate()`, but usually won't need to. It + adds in the state from a list of metrics of the same type as `self`. + (Default is to sum all the variables.) Note that users should not call + `aggregate()`, it is for use by TensorFlow infrastructure. """ def __init__(self, name=None): - self.built = False + self._built = False self._vars = [] + self._initial_values = {} self._updates = [] - self._name = name or self.__class__.__name__ - # TODO(josh11b): Need some way to make sure two Metrics in the same - # Network have distinct names. Maybe we can get a unique name from - # a name/variable scope? - # TODO(josh11b): self._in_graph_mode = context.in_graph_mode() + name = name or self.__class__.__name__ + # Replace things like spaces in name to create a valid scope name. + scope_name = _to_replace.sub("_", name) + # We create the variable scope now to get the unique name that will + # be used as a variable prefix when build() calls add_variable(). + with variable_scope.variable_scope( + scope_name, use_resource=True, reuse=False) as scope: + pos = scope.name.rfind(scope_name) + self._name = name + scope.name[pos + len(scope_name):] + self._scope = scope + if context.in_graph_mode(): + # We make self.call() into a graph callable here, so that we can + # return a single op that performs all of the variable updates. + self._construction_scope = ops.get_default_graph().as_default + self.call = function.defun(self.call) + else: + self._construction_scope = context.eager_mode # ---- API for users ---- def __call__(self, *args, **kwargs): - # TODO(josh11b): If self._in_graph_mode is true, make self.call() into a - # graph callable here, so that variable updates happen without requiring - # a separate fetch. - # TODO(josh11b): Do we need a separate build() method to separate - # initialization from each update? If so, how do we get the arguments - # to it? We *could* just pass in *args and **kwargs... - if not self.built: - # TODO(ashankar): Set up container isolation so there is no chance - # distinct metrics objects accidentally share variables. - # TODO(josh11b): Replace things like spaces in self._name to create - # a valid scope name. + """Returns op to execute to update this metric for these inputs. + + Returns None if eager execution is enabled. + + Args: + *args: + **kwargs: A mini-batch of inputs to the Metric, passed on to `call()`. + """ + if not self._built: with variable_scope.variable_scope( - self._name, use_resource=True, reuse=False): - ret = self.call(*args, **kwargs) - self.built = True - else: - ret = self.call(*args, **kwargs) - return ret + self._scope), self._construction_scope(): + self.build(*args, **kwargs) + self._built = True + return self.call(*args, **kwargs) @property def name(self): @@ -83,11 +127,62 @@ class Metric(object): def variables(self): return self._vars + def init_variables(self): + """Initializes this Metric's variables. + + Should be called after variables are created in the first execution + of `__call__()`. If using graph execution, the return value should be + `run()` in a session before running the op returned by `__call__()`. + (See example above.) + + Returns: + If using graph execution, this returns an op to perform the + initialization. Under eager execution, the variables are reset to their + initial values as a side effect and this function returns None. + """ + if context.in_graph_mode(): + return control_flow_ops.group([v.initializer for v in self._vars]) + for v in self._vars: + v.assign(self._initial_values[v]) + # ---- To be implemented by descendants --- + def build(self, *args, **kwargs): + """Method to create variables. + + Called by `__call__()` before `call()` for the first time. + + Args: + *args: + **kwargs: The arguments to the first invocation of `__call__()`. + `build()` may use the shape and/or dtype of these arguments + when deciding how to create variables. + """ + raise NotImplementedError("Metrics must define a build() member function") + def call(self, *args, **kwargs): - """Accumulates statistics for the metric.""" + """Accumulates statistics for the metric. Users should use __call__ instead. + + Note: This function is executed as a graph function in graph mode. + This means: + a) Operations on the same resource are executed in textual order. + This should make it easier to do things like add the updated + value of a variable to another, for example. + b) You don't need to worry about collecting the update ops to execute. + All update ops added to the graph by this function will be executed. + As a result, code should generally work the same way with graph or + eager execution. + + Args: + *args: + **kwargs: A mini-batch of inputs to the Metric, as passed to + `__call__()`. + """ raise NotImplementedError("Metrics must define a call() member function") + def result(self): # TODO(josh11b): Add an optional summary_writer parameter. + """Computes and returns a final value for the metric.""" + raise NotImplementedError("Metrics must define a result() member function") + # We can support two different strategies of for doing data-parallel # distributed metric computations: # * Put metric variables on the first device and rely on small @@ -123,19 +218,16 @@ class Metric(object): self._vars[i].assign_add(math_ops.add_n([m._vars[i] for m in metrics])) # pylint: enable=protected-access - def result(self): # TODO(josh11b): Add an optional summary_writer parameter. - """Computes and returns a final value for the metric.""" - raise NotImplementedError("Metrics must define a result() member function") - # ---- For use by descendants --- def add_variable(self, name, shape=None, dtype=None, initializer=None): """***Only for use by descendants of Metric***.""" - if self.built: - raise RuntimeError("Can't call add_variable() after a Metric has been " - "built in the first call().") + if self._built: + raise RuntimeError("Can't call add_variable() except in build().") v = variable_scope.get_variable(name, shape, dtype, initializer, trainable=False, use_resource=True) self._vars.append(v) + if context.in_eager_mode(): + self._initial_values[v] = v.value() return v @@ -144,6 +236,21 @@ class Mean(Metric): # TODO(josh11b): Maybe have a dtype argument that defaults to tf.float64? # Or defaults to type of the input if it is tf.float32, else tf.float64? + def __init__(self, name=None, dtype=dtypes.float64): + super(Mean, self).__init__(name=name) + self.dtype = dtype + + def build(self, *args, **kwargs): + # build() does not use call's arguments, by using *args, **kwargs + # we make it easier to inherit from Mean(). + del args, kwargs + self.numer = self.add_variable(name="numer", shape=(), + dtype=self.dtype, + initializer=init_ops.zeros_initializer) + self.denom = self.add_variable(name="denom", shape=(), + dtype=self.dtype, + initializer=init_ops.zeros_initializer) + def call(self, values, weights=None): """Accumulate statistics for computing the mean. @@ -154,31 +261,29 @@ class Mean(Metric): values: Tensor with the per-example value. weights: Optional weighting of each example. Defaults to 1. """ - if not self.built: # False only in the first call(). - self.numer = self.add_variable(name="numer", shape=(), - dtype=dtypes.float64, - initializer=init_ops.zeros_initializer) - self.denom = self.add_variable(name="denom", shape=(), - dtype=dtypes.float64, - initializer=init_ops.zeros_initializer) if weights is None: self.denom.assign_add( - math_ops.cast(array_ops.size(values), dtypes.float64)) + math_ops.cast(array_ops.identity(array_ops.size(values)), self.dtype)) values = math_ops.reduce_sum(values) - self.numer.assign_add(math_ops.cast(values, dtypes.float64)) + self.numer.assign_add(math_ops.cast(values, self.dtype)) else: - weights = math_ops.cast(weights, dtypes.float64) + weights = math_ops.cast(weights, self.dtype) self.denom.assign_add(math_ops.reduce_sum(weights)) - values = math_ops.cast(values, dtypes.float64) * weights + values = math_ops.cast(values, self.dtype) * weights self.numer.assign_add(math_ops.reduce_sum(values)) def result(self): - return self.numer / self.denom + t = self.numer / self.denom + summary_ops.scalar(name=self.name, tensor=t) + return t class Accuracy(Mean): """Calculates how often `predictions` matches `labels`.""" + def __init__(self, name=None, dtype=dtypes.float64): + super(Accuracy, self).__init__(name=name, dtype=dtype) + def call(self, labels, predictions, weights=None): """Accumulate accuracy statistics. diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py index 089bad5a0e3049543bdc09b571319262a734809f..b945e97a0049441d356f41e4d19fe6f01836ec40 100644 --- a/tensorflow/contrib/eager/python/metrics_test.py +++ b/tensorflow/contrib/eager/python/metrics_test.py @@ -18,8 +18,16 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import tempfile + from tensorflow.contrib.eager.python import metrics +from tensorflow.contrib.summary import summary_ops +from tensorflow.contrib.summary import summary_test_util +from tensorflow.python.eager import context from tensorflow.python.eager import test +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops +from tensorflow.python.training import training_util class MetricsTest(test.TestCase): @@ -30,6 +38,31 @@ class MetricsTest(test.TestCase): m(1000) m([10000.0, 100000.0]) self.assertEqual(111111.0/6, m.result().numpy()) + self.assertEqual(dtypes.float64, m.dtype) + self.assertEqual(dtypes.float64, m.result().dtype) + + def testInitVariables(self): + m = metrics.Mean() + m([1, 10, 100, 1000]) + m([10000.0, 100000.0]) + self.assertEqual(111111.0/6, m.result().numpy()) + m.init_variables() + m(7) + self.assertEqual(7.0, m.result().numpy()) + + def testWriteSummaries(self): + m = metrics.Mean() + m([1, 10, 100]) + training_util.get_or_create_global_step() + logdir = tempfile.mkdtemp() + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name="t0").as_default(), summary_ops.always_record_summaries(): + m.result() # As a side-effect will write summaries. + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 37.0) def testWeightedMean(self): m = metrics.Mean() @@ -37,6 +70,14 @@ class MetricsTest(test.TestCase): m([500000, 5000, 500]) # weights of 1 each self.assertNear(535521/4.5, m.result().numpy(), 0.001) + def testMeanDtype(self): + # Can override default dtype of float64. + m = metrics.Mean(dtype=dtypes.float32) + m([0, 2]) + self.assertEqual(1, m.result().numpy()) + self.assertEqual(dtypes.float32, m.dtype) + self.assertEqual(dtypes.float32, m.result().dtype) + def testAccuracy(self): m = metrics.Accuracy() m([0, 1, 2, 3], [0, 0, 0, 0]) # 1 correct @@ -45,6 +86,8 @@ class MetricsTest(test.TestCase): m([6], [6]) # 1 correct m([7], [2]) # 0 correct self.assertEqual(3.0/8, m.result().numpy()) + self.assertEqual(dtypes.float64, m.dtype) + self.assertEqual(dtypes.float64, m.result().dtype) def testWeightedAccuracy(self): m = metrics.Accuracy() @@ -56,6 +99,58 @@ class MetricsTest(test.TestCase): m([7], [2]) # 0 correct, weight 1 self.assertEqual(2.5/5, m.result().numpy()) + def testAccuracyDtype(self): + # Can override default dtype of float64. + m = metrics.Accuracy(dtype=dtypes.float32) + m([0, 0], [0, 1]) + self.assertEqual(0.5, m.result().numpy()) + self.assertEqual(dtypes.float32, m.dtype) + self.assertEqual(dtypes.float32, m.result().dtype) + + def testTwoMeans(self): + # Verify two metrics with the same class and name don't + # accidentally share state. + m1 = metrics.Mean() + m1(0) + m2 = metrics.Mean() + m2(2) + self.assertAllEqual(0.0, m1.result()) + self.assertAllEqual(2.0, m2.result()) + + def testNamesWithSpaces(self): + # Verify two metrics with the same class and name don't + # accidentally share state. + m1 = metrics.Mean("has space") + m1(0) + self.assertEqual(m1.name, "has space") + self.assertEqual(m1.numer.name, "has_space/numer:0") + + def testGraph(self): + with context.graph_mode(), self.test_session() as sess: + m = metrics.Mean() + p = array_ops.placeholder(dtypes.float32) + accumulate = m(p) + init_op = m.init_variables() + init_op.run() + sess.run(accumulate, feed_dict={p: [1, 10, 100]}) + sess.run(accumulate, feed_dict={p: 1000}) + sess.run(accumulate, feed_dict={p: [10000, 100000]}) + self.assertAllEqual(m.result().eval(), 111111.0/6) + # Second init resets all the variables. + init_op.run() + sess.run(accumulate, feed_dict={p: 7}) + self.assertAllEqual(m.result().eval(), 7) + + def testTwoMeansGraph(self): + # Verify two metrics with the same class and name don't + # accidentally share state. + with context.graph_mode(): + m1 = metrics.Mean() + m1(0) + with self.assertRaises(ValueError): + m2 = metrics.Mean() + m2(2) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py index bebc595df07dbd3a0ecfe5c93749f13332805539..5b53a597f20a1cd0ba9be7f1d3a89e117cde66e8 100644 --- a/tensorflow/contrib/eager/python/network.py +++ b/tensorflow/contrib/eager/python/network.py @@ -19,13 +19,167 @@ from __future__ import division from __future__ import print_function import collections -import uuid - -import six +import os +import weakref +from tensorflow.python.eager import context +from tensorflow.python.estimator import util as estimator_util from tensorflow.python.framework import ops from tensorflow.python.layers import base from tensorflow.python.ops import variable_scope +from tensorflow.python.training import checkpoint_utils +from tensorflow.python.training import saver as saver_lib +from tensorflow.python.training import training_util + +# pylint: disable=protected-access +# Explanation for protected-access disable: Network has lots of same-class and +# parent-class references across different objects, and some to private +# functions in base.py which should be reused. + + +_DeferredRestoration = collections.namedtuple( + + "_DeferredRestoration", + [ + # The map_func to use (either user-specified or the default). + "map_func", + # Boolean, True if the user specified an explicit map_func, for error + # messages. + "map_func_is_user", + # A mapping from checkpoint names to initial values of not-yet-created + # variables which should be restored. These values come from parsing a + # checkpoint. + "checkpointed_variables_to_restore", + # A mapping from checkpoint name to variable objects of variables which + # have already been restored, for error checking. + "restored_variables", + # The session to restore with (if in graph mode). + "session", + # Names of the Network where the restore was requested, for error + # messages. + "network_name", + "network_scope_name" + ]) + + +def _default_naming_conflict_error_message( + mapped_name, first_variable, second_variable, + network_name, network_scope_name): + return ( + ("The default checkpoint variable name mapping strategy for Network " + "'%s' resulted in a naming conflict. We attempted to strip off the " + "variable prefix for the Network ('%s'), but this resulted in two " + "variables named '%s' (originally '%s' and '%s'). This should only " + "happen when using variable sharing (i.e. the Network contains Networks " + "or Layers which were first added to another Network, and therefore " + "have that Network's variable prefix). One solution is to pass " + "`map_func=lambda n: n` to Network.save and Network.restore to use " + "fully qualified variable names in the checkpoint, although this will " + "require that the variable prefix of the Network being restored into " + "is also '%s'. You may alternatively write an arbitrary mapping.") + % ( + network_name, network_scope_name, mapped_name, + first_variable._shared_name, + second_variable._shared_name, network_scope_name + )) + + +def _restore_custom_map_func_error_message( + mapped_name, first_variable, second_variable, + network_name, network_scope_name): + return ( + ("The map_func passed to Network.restore for the Network '%s' " + "resulted in two variables named '%s' (originally '%s' and '%s'). Since " + "this is also an error on Network.save, this Network was " + "probably not saved with this map_func. Note that map_func " + "always maps from full variable names to checkpoint names; " + "there is no need to specify an inverse mapping.\n\n" + "Try stripping less from the variable names, or renaming parts " + "of the Network. For reference, variables created by sub-Layers " + "of this Network are prefixed with '%s', but if they are " + "re-used after being added to another Network they will have " + "that Network's full variable prefix instead.") % ( + network_name, mapped_name, + first_variable._shared_name, + second_variable._shared_name, + network_scope_name)) + + +def _make_custom_getter_for_deferred_restorations(): + """Returns a custom getter which searches `deferred_restorations`. + + Returns: A tuple of (_custom_getter, deferred_restorations) + _custom_getter: The getter which should be added to variable_scopes where + variables will be created. + deferred_restorations: A list for _DeferredRestoration objects. Typically + empty when the getter is set, and expanded as deferred restorations are + requested. All new deferred restorations should be appended to the end of + the list, where they will have priority over older deferred restorations. + """ + deferred_restorations = [] + + def _custom_getter(getter, name, shape=None, dtype=None, + initializer=None, + *args, **kwargs): + """A custom getter which processes deferred restorations.""" + # Iterate over restorations, newest first (newer restorations will take + # precedence over older restorations, just like with immediate restorations + # into existing variables). + delayed_restoration = None + found_value = False + value_to_restore = None + for delayed_restoration in reversed( + deferred_restorations): + checkpoint_name = delayed_restoration.map_func(name) + if (checkpoint_name + in delayed_restoration.checkpointed_variables_to_restore): + found_value = True + value_to_restore = ( + delayed_restoration.checkpointed_variables_to_restore[ + checkpoint_name]) + if found_value: + break + # value_to_restore may be False because this variable is not in any + # checkpoint we are restoring, or None because we have explicitly set it to + # None when it was previously fetched. In either case, we don't need to + # set an initializer. + if found_value and value_to_restore is not None: + initializer = value_to_restore + shape = None + variable = getter(name, shape=shape, dtype=dtype, initializer=initializer, + *args, **kwargs) + if found_value and value_to_restore is not None: + # Mark as already restored from this checkpoint. + delayed_restoration.checkpointed_variables_to_restore[ + checkpoint_name] = None + if context.in_graph_mode(): + delayed_restoration.session.run(variable.initializer) + if found_value: + # Error checking should run even if we've already restored a value. + if delayed_restoration.restored_variables.setdefault( + checkpoint_name, variable) is not variable: + # Naming conflict. We've tried to initialize two variables with the + # same value from the checkpoint. + if delayed_restoration.map_func_is_user: + raise ValueError( + _restore_custom_map_func_error_message( + mapped_name=checkpoint_name, + first_variable=delayed_restoration.restored_variables[ + checkpoint_name], + second_variable=variable, + network_name=delayed_restoration.network_name, + network_scope_name=delayed_restoration.network_scope_name)) + else: + raise ValueError( + _default_naming_conflict_error_message( + mapped_name=checkpoint_name, + first_variable=delayed_restoration.restored_variables[ + checkpoint_name], + second_variable=variable, + network_name=delayed_restoration.network_name, + network_scope_name=delayed_restoration.network_scope_name)) + return variable + return _custom_getter, deferred_restorations class Network(base.Layer): @@ -34,28 +188,163 @@ class Network(base.Layer): TODO(josh11b,ashankar): - Should "trainable" be changeable on the Network object? - Do we allow add_variable in Network? - - Layer.name and Layer.variables.names are not in sync today - d = tf.layers.Dense(1) - d(tf.constant([[1.]])) - print(d.name) - print(d.variables) - - Note that name provided to __init__ is only for error messages? - - Detect layers used in __call__ that weren't registered with add_layer. + - Detect layers used in __call__ that weren't registered with track_layer. - Convert inputs to __call__ to tensors. - Prevent variables from being created after the first __call__? (Think about restoring from a checkpoint). - - Save & restore """ def __init__(self, name=None): + if isinstance(name, variable_scope.VariableScope): + raise ValueError("VariableScopes are not valid Network names.") + if name is not None and "/" in name: + raise ValueError( + "Forward slashes ('/') are not allowed in Network names.") super(Network, self).__init__(name=name) - self._container = uuid.uuid4().hex - self._layers = collections.OrderedDict() + self._layers = [] + self._sub_layer_name_uids = collections.defaultdict(int) + # Initially None, but set to False for networks which are first built as + # top-level. + self._first_parent = None # A weak reference to our first parent. + self._non_network_sublayers = [] + self._owned_layers = {} + # The scope to use if we end up without a parent. + self._default_parent_variable_scope = variable_scope.get_variable_scope() + self._custom_getter, self._deferred_restorations = ( + _make_custom_getter_for_deferred_restorations()) + + def _init_set_name(self, name): + # Anonymous Networks (name=None) defer setting a final name until they are + # (1) added to another Network, or (2) built/called (where (2) is only used + # for a "top level" network). + # + # However, if we were provided an explicit name (name is not None), that + # will always be the final name of the Network; if it turns out not to be + # unique or if variable names can't be prefixed by it we will throw an + # error. + self._name = name + self._base_name = None + + def _finalize_name(self, parent_network): + if not self._name: + if not parent_network: + name_uid_map = base._get_default_graph_uid_map() + else: + name_uid_map = parent_network._sub_layer_name_uids + # Were were not passed a name explicitly (or it was blank), so this is an + # anonymous Network. We make up a unique name. + if parent_network: + avoid_names = parent_network._owned_layers + else: + avoid_names = None + self._name, self._base_name = self._make_unique_name( + name_uid_map=name_uid_map, avoid_names=avoid_names) + if self._first_parent is None or (self._first_parent # False = no parent + and self._first_parent() is None): + # Save a pointer to the parent Network so that we can later check that the + # scope name we get is correct. + if not parent_network: + self._first_parent = parent_network + else: + self._first_parent = weakref.ref(parent_network) + + def _set_scope(self, scope=None): + if self._scope is None: + if not self._first_parent: + first_parent = self._first_parent + else: + first_parent = self._first_parent() + if first_parent is None: + # If we were never added to another Network, or that Network has beed + # garbage collected before being called, then we're a top-level Network. + self._finalize_name( + # Use False to make sure the value sticks and we don't inherit a + # parent if we're added to a network later. + parent_network=False) + if scope is not None: + raise ValueError("Networks may not be created with explicit scopes.") + if first_parent: + first_parent._set_scope() + parent_scope = first_parent._scope + else: + parent_scope = self._default_parent_variable_scope + with variable_scope.variable_scope(parent_scope): + # Make sure variables with this prefix will be unique. + with variable_scope.variable_scope( + None, use_resource=True, default_name=self._name) as scope: + self._scope = scope + scope_name = scope.name + suffix_start = scope_name.rfind("/") + 1 + # rfind is -1 if there is no slash in the string, in which case the + # suffix starts at the beginning of the string (there is no prefix). + scope_suffix = scope_name[suffix_start:] + scope_prefix = scope_name[:suffix_start] + if scope_suffix != self._name: + raise ValueError( + ("A Network named '%s' already exists (or a variable_scope was " + "created with this name). Names must be unique.") % ( + self._name,)) + if (first_parent + and scope_prefix[:-1] != first_parent._scope.name): + raise ValueError( + ("Network variable names must match a nesting of sub-Network " + "names. Expected prefix '%s' from parent network, but got " + "'%s' when attempting to create a variable_scope for Network " + "'%s'. Likely an explicit variable_scope was inserted into " + "the nesting.") % ( + first_parent._scope.name, + scope_prefix[:-1], + self._name)) + elif not first_parent and scope_prefix: + # For the case when this Network is not nested inside any other + # Network, but is in a variable_scope. This is an error for now. + raise ValueError( + "Creating Networks inside named variable_scopes is currently " + "not supported (to ensure that variable names match the names " + "of Networks in which they were first created). To set " + "options, try `with tf.variable_scope(''):`. If this " + "limitation bothers you, please file a feature request.") + for non_network_sublayer in self._non_network_sublayers: + self._set_scope_for_nonnetwork_sublayer(non_network_sublayer) + + def _set_scope_for_nonnetwork_sublayer(self, sublayer): + if sublayer._scope is None: + if sublayer._first_parent is None: + constituent_first_parent = None + else: + constituent_first_parent = sublayer._first_parent() + if constituent_first_parent: + constituent_first_parent._set_scope() + parent_scope = constituent_first_parent._scope + else: + self._finalize_name(False) + raise ValueError( + ("The parent of a Layer added to Network %s was garbage collected " + "before the Layer was built. If this limitation bothers you " + "please, file a feature request.") % (self.name,)) + with variable_scope.variable_scope(parent_scope): + # Horrid hack to make Layer variable names which are direct + # sub-layers of Networks conform to the Network variable naming + # conventions. + with variable_scope.variable_scope( + None, use_resource=True, + default_name=sublayer.name) as sub_scope: + sublayer._scope = sub_scope + + @base.Layer.name.getter + def name(self): + if self._name is None: + raise ValueError( + "The network does not yet have a final name, but a name was " + "requested for it. Networks get a name when they are added to " + "another Network via track_layer, or when they are first " + "called/built.") + return self._name - def add_layer(self, layer): - """Add a Layer to this Network. + def track_layer(self, layer): + """Track a Layer in this Network. - `Network` requires that all `Layer`s used in `call()` be added so that the + `Network` requires that all `Layer`s used in `call()` be tracked so that the `Network` can export a complete list of variables. Args: @@ -66,29 +355,60 @@ class Network(base.Layer): Raises: RuntimeError: If __init__ has not been called. - TypeError: If layer is the wrong type. - ValueError: If a layer with the same name has already been added. + TypeError: If `layer` is the wrong type. + ValueError: If a `Layer` with the same name has already been added. """ if not hasattr(self, "_layers"): raise RuntimeError("Need to call Network.__init__ before adding layers") if not isinstance(layer, base.Layer): raise TypeError( - "Network.add_layer() passed type %s, not a tf.layers.Layer" % + "Network.track_layer() passed type %s, not a tf.layers.Layer" % (type(layer),)) - if layer.name in self._layers: - if self._layers[layer.name] is layer: - return layer - raise ValueError( - "Attempt to add two Layers with the name '%s' to the same Network " - "'%s'" % (layer.name, self.name)) - self._layers[layer.name] = layer + if isinstance(layer, Network): + layer._finalize_name(parent_network=self) + else: + # `layer` is a non-Network, so it hasn't been named to follow Network + # conventions for contained Layers (i.e. the same conventions as for + # sub-Networks). This renaming is necessary to isolate Network variable + # naming from Layers constructed outside the Network and never added to it + # (because Layers are named globally). + if not layer.built: + if not hasattr(layer, "_first_parent"): + dereferenced_layer_first_parent = None + else: + dereferenced_layer_first_parent = layer._first_parent() + if dereferenced_layer_first_parent is None: + if layer._name != layer._base_name: + # If name and base_name do not match, then this Layer used anonymous + # naming and we have to rename it. Otherwise there's an explicit + # name, and we should respect it (subject to error checking). + layer._name, layer._base_name = layer._make_unique_name( + name_uid_map=self._sub_layer_name_uids, + avoid_names=self._owned_layers) + layer._first_parent = weakref.ref(self) + self._non_network_sublayers.append(layer) + if (not layer.built + and layer._first_parent + and self is layer._first_parent()): + if layer.name in self._owned_layers: + if self._owned_layers[layer.name] is layer: + return layer + raise ValueError( + "Attempt to add two Layers with the name '%s' to the same Network." + % (layer.name)) + self._owned_layers[layer.name] = layer + self._layers.append(layer) return layer def get_layer(self, name=None, index=None): """Get a contained `tf.layers.Layer` either by name or index. Args: - name: String matching one of the names of a contained `Layer`. + name: String matching one of the names of a contained `Layer`. Note that + the names of `Layer`s added to `Network`s may not be unique when doing + layer sharing (i.e. adding a `Layer` to this `Network` which was already + added to another `Network`). The lowest index `Layer` with a matching + name will be returned. index: Integer in [0, number of layers). Layers are assigned an index by the order they are added. @@ -96,19 +416,25 @@ class Network(base.Layer): A `tf.layers.Layer` object. Raises: - ValueError: If neither or both of 'index' or 'name' is specified. + ValueError: If neither or both of 'index' or 'name' is specified, or the + lookup failed. """ if index is not None: if name is not None: raise ValueError("Exactly one of 'index' or 'name' must be provided") if len(self._layers) <= index: - raise ValueError("Was asked to retrieve layer at index " + - str(index) + " but model only has " + str( - len(self._layers)) + " layers.") - return list(self._layers.values())[index] - if name is None: - raise ValueError("Exactly one of 'index' or 'name' must be provided") - return self._layers[index] + raise ValueError("Was asked to retrieve layer at index " + str(index) + + " but model only has " + str(len(self._layers)) + + " layers.") + else: + return self._layers[index] + else: + if not name: + raise ValueError("Provide either a layer name or layer index.") + for layer in self._layers: + if layer.name == name: + return layer + raise ValueError("No such layer: " + name) # The following methods are for implementing the Layer interface. @@ -118,21 +444,21 @@ class Network(base.Layer): # variables in the case of shared layers/variables that appear in # multiple places in the Network? weights = [] - for layer in six.itervalues(self._layers): + for layer in self._layers: weights += layer.weights return weights @property def trainable_weights(self): weights = [] - for layer in six.itervalues(self._layers): + for layer in self._layers: weights += layer.trainable_weights return weights @property def non_trainable_weights(self): weights = [] - for layer in six.itervalues(self._layers): + for layer in self._layers: weights += layer.non_trainable_weights return weights @@ -151,7 +477,7 @@ class Network(base.Layer): @property def layers(self): - return self._layers.values() + return self._layers def add_variable(self, name, shape, dtype=None, initializer=None, regularizer=None, trainable=True, constraint=None): @@ -160,40 +486,316 @@ class Network(base.Layer): "at https://github.com/tensorflow/tensorflow/issues/new if this is " "important to you") - def __call__(self, inputs, *args, **kwargs): - # TODO(josh11b,ashankar,agarwal): Can we reduce the number of context - # managers here and/or move some of the work into the constructor - # for performance reasons? - with ops.container(self._container): - with variable_scope.variable_scope(variable_scope.get_variable_scope(), - use_resource=True): - return super(Network, self).__call__(inputs, *args, **kwargs) + def _strip_variable_prefix(self, original_variable_name): + """The default map_func for saving or restoring variables. + + Strips the variable prefix for the Network on which save/restore was called, + and leaves other variable names fully qualified in the checkpoint. + + Args: + original_variable_name: The _shared_name of the variable (no :0 + suffix) to map. + Returns: + The checkpoint name of the variable. + """ + scope_name_with_slash = self.scope_name + "/" + if original_variable_name.startswith(scope_name_with_slash): + return original_variable_name[len(scope_name_with_slash):] + else: + return original_variable_name + + def save(self, save_path, global_step=None, map_func=None): + """Save variables from the Network to a checkpoint. + + Args: + save_path: Either a checkpoint prefix or the name of a directory to save + the checkpoint in (in which case the checkpoint will be named based on + the Network name). + global_step: The global step to use when naming the checkpoint. If None + (default), we will first try to get the default global step. If that + fails because no default global step exists, then the checkpoint is + created without a global step suffix. + map_func: A function mapping fully qualified variable names + (e.g. 'my_network_1/dense_1/kernel') to names in the checkpoint. By + default (if `map_func=None`), the variable prefix for the network being + restored (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped + and all other variable names (shared with other Networks) are left + unchanged. + Returns: + The checkpoint prefix for the saved checkpoint, which may be passed to + `Network.restore`. + Raises: + ValueError: If the Network has not yet been called, or if map_func results + in a name collision. + """ + if not self.built: + raise ValueError( + "Attempt to save the Network before it was first called. This means " + "variables have not yet been created, so there is nothing to save.") + self._set_scope() # scope_name should be available to map_funcs + if global_step is None: + global_step = training_util.get_global_step() + if os.path.isdir(save_path): + # If we were passed a directory, default to naming based on the Network + # name. + save_path = os.path.join(save_path, self.name) + user_map_func = map_func + if map_func is None: + map_func = self._strip_variable_prefix + variable_map = {} + for variable in self.variables: + mapped_name = map_func(variable._shared_name) + if variable_map.setdefault(mapped_name, variable) is not variable: + if user_map_func is None: + # Instead of erroring out, we could just re-try and silently use the + # full variable names in the checkpoint. This could be odd for deeply + # nested sub-Networks (since the full prefix from the nesting would + # get added), so for now we'll let the user deal with this case. + raise ValueError(_default_naming_conflict_error_message( + mapped_name=mapped_name, + first_variable=variable_map[mapped_name], + second_variable=variable, + network_name=self.name, + network_scope_name=self.scope_name)) + else: + # The user passed their own problematic map_func. + raise ValueError( + ("The map_func passed to Network.save for the Network '%s' " + "resulted in two variables named '%s' ('%s' and '%s'). Try " + "stripping less from the variable names, or renaming parts of " + "the Network. For reference, variables created by sub-Layers of " + "this Network are prefixed with '%s', but if they are re-used " + "after being added to another Network, they will have that " + "Network's full variable prefix instead.") % ( + self.name, mapped_name, + variable_map[mapped_name]._shared_name, + variable._shared_name, + self.scope_name)) + if context.in_eager_mode(): + sess = None + else: + sess = ops.get_default_session() + return saver_lib.Saver(variable_map).save( + sess=sess, save_path=save_path, write_meta_graph=False, + global_step=global_step) + + def _restore_existing_variables(self, save_path, map_func, user_map_func): + """Use a standard Saver to restore existing variables from a checkpoint. + + Args: + save_path: The checkpoint prefix or directory to read from. + map_func: The function to use when mapping from variable names to + checkpoint names. + user_map_func: The original map_func passed by the user, for error + checking. + Returns: + A dictionary mapping from checkpoint names to variable objects which have + been restored (for bookkeeping to avoid deferred restorations on these + variables). + Raises: + ValueError: If there is a name collision. + """ + existing_variables_by_checkpoint_name = {} + for variable in self.variables: + checkpoint_name = map_func(variable._shared_name) + if existing_variables_by_checkpoint_name.setdefault( + checkpoint_name, variable) is not variable: + if user_map_func is None: + raise ValueError(_default_naming_conflict_error_message( + mapped_name=checkpoint_name, + first_variable=existing_variables_by_checkpoint_name[ + checkpoint_name], + second_variable=variable, + network_name=self.name, + network_scope_name=self.scope_name)) + else: + raise ValueError(_restore_custom_map_func_error_message( + mapped_name=checkpoint_name, + first_variable=existing_variables_by_checkpoint_name[ + checkpoint_name], + second_variable=variable, + network_name=self.name, + network_scope_name=self.scope_name)) + if existing_variables_by_checkpoint_name: + if context.in_eager_mode(): + sess = None + else: + sess = ops.get_default_session() + saver_lib.Saver(var_list=existing_variables_by_checkpoint_name).restore( + sess=sess, save_path=save_path) + return existing_variables_by_checkpoint_name + + def _set_restore_on_create(self, save_path, map_func, user_map_func, + existing_variables_by_checkpoint_name): + """If necessary, request deferred restorations of variables.""" + checkpoint_reader = checkpoint_utils.load_checkpoint(save_path) + checkpointed_variables_to_restore = {} + for checkpoint_name, _ in checkpoint_utils.list_variables(save_path): + if checkpoint_name in existing_variables_by_checkpoint_name: + # This variable was already created and restored. + continue + # Save the variable for later restoration in a custom getter. + checkpointed_variables_to_restore[checkpoint_name] = ( + checkpoint_reader.get_tensor(checkpoint_name)) + # Only set a deferred restoration if there are checkpoint variables which + # have not been assigned to existing variables. Note that this loses out on + # some opportunity for error checking, but avoids creating + # _DeferredRestoration objects once a Network has been built (so that + # restoring in a loop does not take increasing amounts of memory). + if checkpointed_variables_to_restore: + if context.in_eager_mode(): + sess = None + else: + sess = ops.get_default_session() + # We need a name for error messages. If we haven't been added to another + # Network yet, we're top-level. + self._finalize_name(False) + self._set_scope() + # Save a record of this restoration for use in the custom getter. + deferred_restoration = _DeferredRestoration( + map_func=map_func, + map_func_is_user=(user_map_func is not None), + checkpointed_variables_to_restore=checkpointed_variables_to_restore, + restored_variables={}, + session=sess, + network_name=self.name, + network_scope_name=self.scope_name) + self._deferred_restorations.append(deferred_restoration) + # Add the deferred registration to non-Network children, and request that + # Networks propagate the request to their children. + self._add_deferred_restoration(deferred_restoration) + + def _add_deferred_restoration(self, deferred_restoration): + """Add a deferred restoration to this Network and all children. + + Restorations which are requested later have higher priority, and the highest + priority matching restoration is applied to a variable when it is created. + + Args: + deferred_restoration: A _DeferredRestoration object. + """ + # Networks don't create variables at the moment, so this append isn't + # strictly necessary. We could get by with only adding deferred restorations + # to non-Network Layers. + self._set_scope() + # We use set_custom_getter because it avoids recursively calling up the + # variable_scope tree. We've done the tree traversal ourselves and have + # added the request to each Layer which needs it. + self._scope.set_custom_getter(self._custom_getter) + self._deferred_restorations.append(deferred_restoration) + for layer in self.layers: + if isinstance(layer, Network): + # For Networks, request that they propagate this deferred restoration + # to all of their children recursively. + layer._add_deferred_restoration(deferred_restoration) + else: + # For non-Network Layers, make sure they have a deferred restoration + # queue and a custom getter, then add our request to it. + if not hasattr(layer, "_custom_getter"): + assert not hasattr(layer, "_deferred_restorations") + layer._custom_getter, layer._deferred_restorations = ( + _make_custom_getter_for_deferred_restorations()) + self._set_scope_for_nonnetwork_sublayer(layer) + layer._scope.set_custom_getter(layer._custom_getter) + layer._deferred_restorations.append(deferred_restoration) + + def restore(self, save_path, map_func=None): + """Restore the Network from a checkpoint. + + If variables have already been created (typically when some or all of the + `Network` is built), they are assigned values from the checkpoint + immediately, overwriting any existing values (in graph mode the default + session is used for the assignments). + + If there are checkpoint entries which do not correspond to any existing + variables in the `Network`, these values are saved for deferred restoration; + their initial values will be the checkpointed values once they are + created. Requests for multiple deferred restorations behave the same way as + immediate restorations, in that later requests will take priority over + earlier requests relevant to the same variable. + + If this `Network` shares `Layer`s with another network, those `Layer`s will + also have their variables restored from the checkpoint. + + Args: + save_path: The return value of `Network.save`, or a directory to search + for a checkpoint. + map_func: A function mapping fully qualified variable names + (e.g. 'my_network_1/dense_1/kernel') to names in the checkpoint. By + default (if `map_func=None`), the variable prefix for the network being + restored (`Network.scope_name + '/'`, e.g. 'my_network_1/') is stripped + and all other variable names (shared with other Networks) are left + unchanged. Note that this is the _same_ map_func as `Network.save`, not + an inverse mapping. + """ + self._finalize_name(parent_network=False) + self._set_scope() # scope_name should be available to map_funcs + if os.path.isdir(save_path): + # If we don't have a name yet, set no parent. + save_path = os.path.join(save_path, self.name) + user_map_func = map_func + if map_func is None: + map_func = self._strip_variable_prefix + # Step one is to restore any existing variables from the checkpoint. + existing_variables_by_checkpoint_name = self._restore_existing_variables( + save_path=save_path, + map_func=map_func, + user_map_func=user_map_func) + # Step two is to set a custom getter which restores variables on creation, + # for those variables which have not been added to sub-Layers yet. + self._set_restore_on_create( + save_path=save_path, + map_func=map_func, + user_map_func=user_map_func, + existing_variables_by_checkpoint_name=( + existing_variables_by_checkpoint_name)) # TODO(josh11b): Support other Layer methods needed for graph mode, such as for # losses and updates class Sequential(Network): - """Represents a linear sequence of Layers. + """Represents a linear sequence of Layers or functions. - The output of each layer is provided as the input to the next. + The output of each layer/function is provided as the input to the next. The inputs passed to `__call__` are passed to the inputs of the first Layer, and it returns the outputs of the last Layer. Args: - layers: An optional sequence of tf.layers.Layer objects. + layers_funcs: An optional sequence where each element is either a + tf.layers.Layer object or a callable. name: An optional string name to use for this Network. """ - def __init__(self, layers=None, name=None): + def __init__(self, layers_funcs=None, name=None): super(Sequential, self).__init__(name=name) - if layers: - for l in layers: - self.add_layer(l) + self._layers_funcs = [] + if layers_funcs: + for l in layers_funcs: + self.add(l) + + def add(self, layer_func): + if isinstance(layer_func, base.Layer): + args = estimator_util.fn_args(layer_func.call) + self.track_layer(layer_func) + elif callable(layer_func): + args = estimator_util.fn_args(layer_func) + else: + raise TypeError( + "Sequential.add() takes only tf.layers.Layer objects or callables; " + "not '%s' of type '%s'." % (layer_func, type(layer_func))) + self._layers_funcs.append((("training" in args), layer_func)) - def call(self, inputs): + def call(self, inputs, training=None): """Call each Layer in the order they were added.""" # TODO(josh11b): Support "mode" and maybe other arguments - for l in self.layers: - inputs = l(inputs) + if training is None: + for _, l in self._layers_funcs: + inputs = l(inputs) + else: + for has_training_arg, l in self._layers_funcs: + if has_training_arg: + inputs = l(inputs, training) + else: + inputs = l(inputs) return inputs diff --git a/tensorflow/contrib/eager/python/network_test.py b/tensorflow/contrib/eager/python/network_test.py index f0dcae85ee139405784a70c2d3704b0bbcf9e4dd..c621f527c28306131bdba56d8427eaa787ba150b 100644 --- a/tensorflow/contrib/eager/python/network_test.py +++ b/tensorflow/contrib/eager/python/network_test.py @@ -16,18 +16,27 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gc + from tensorflow.contrib.eager.python import network from tensorflow.python.eager import test from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.layers import core +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.training import training_util # pylint: disable=not-callable class MyNetwork(network.Network): - def __init__(self): - super(MyNetwork, self).__init__(name="abcd") - self.l1 = self.add_layer(core.Dense(1, use_bias=False)) + def __init__(self, name=None): + super(MyNetwork, self).__init__(name=name) + self.l1 = self.track_layer(core.Dense(1, use_bias=False)) def call(self, x): return self.l1(x) @@ -35,6 +44,30 @@ class MyNetwork(network.Network): class NetworkTest(test.TestCase): + def _save_modify_load_network_built(self, net, global_step=None): + checkpoint_directory = self.get_temp_dir() + checkpoint_path = net.save( + save_path=checkpoint_directory, global_step=global_step) + input_value = constant_op.constant([[42.0]]) + original_output = self.evaluate(net(input_value)) + for var in net.variables: + self.evaluate(var.assign(var + 1.)) + self.assertGreater( + self.evaluate(net(input_value)), + original_output) + # Either the returned explicit checkpoint path or the directory should work. + net.restore(save_path=checkpoint_directory) + self.assertAllEqual( + original_output, + self.evaluate(net(input_value))) + for var in net.variables: + self.evaluate(var.assign(var + 2.)) + net.restore(save_path=checkpoint_path) + self.assertAllEqual( + original_output, + self.evaluate(net(input_value))) + + @test_util.run_in_graph_and_eager_modes() def testTrainableAttribute(self): net = network.Network() self.assertTrue(net.trainable) @@ -42,41 +75,934 @@ class NetworkTest(test.TestCase): net.trainable = False self.assertTrue(net.trainable) + @test_util.run_in_graph_and_eager_modes() def testNetworkCall(self): - net = MyNetwork() + net = MyNetwork(name="abcd") net(constant_op.constant([[2.0]])) # Force variables to be created. self.assertEqual(1, len(net.trainable_variables)) - net.trainable_variables[0].assign([[17.0]]) + self.evaluate(net.trainable_variables[0].assign([[17.0]])) # TODO(josh11b): Support passing Python values to networks. result = net(constant_op.constant([[2.0]])) - self.assertEqual(34.0, result.numpy()) + self.assertEqual(34.0, self.evaluate(result)) + + @test_util.run_in_graph_and_eager_modes() + def testNetworkSaveRestoreAlreadyBuilt(self): + net = MyNetwork(name="abcd") + with self.assertRaisesRegexp( + ValueError, "Attempt to save the Network before it was first called"): + net.save(self.get_temp_dir()) + net(constant_op.constant([[2.0]])) + self.evaluate(net.trainable_variables[0].assign([[17.0]])) + self._save_modify_load_network_built(net, global_step=None) + self._save_modify_load_network_built(net, global_step=10) + + @test_util.run_in_graph_and_eager_modes() + def testSaveRestoreDefaultGlobalStep(self): + net = MyNetwork(name="abcd") + net(constant_op.constant([[2.0]])) + self.evaluate(net.variables[0].assign([[3.]])) + default_global_step = training_util.get_or_create_global_step() + self.evaluate(default_global_step.assign(4242)) + save_path = net.save(self.get_temp_dir()) + self.assertIn("abcd-4242", save_path) + + @test_util.run_in_graph_and_eager_modes() + def testNetworkSaveAndRestoreIntoUnbuilt(self): + save_dir = self.get_temp_dir() + net1 = MyNetwork() + test_input = constant_op.constant([[2.0]]) + net1(test_input) + self.evaluate(net1.trainable_variables[0].assign([[17.0]])) + save_path = net1.save(save_dir) + # With a pre-build restore we should have the same value. + net2 = MyNetwork() + net2.restore(save_path) + self.assertAllEqual(self.evaluate(net1(test_input)), + self.evaluate(net2(test_input))) + self.assertIsNot(net1.variables[0], net2.variables[0]) + self.assertAllEqual(self.evaluate(net1.variables[0]), + self.evaluate(net2.variables[0])) + + @test_util.run_in_graph_and_eager_modes() + def testLoadIntoUnbuiltSharedLayer(self): + + class Owner(network.Network): + + def __init__(self, name=None): + super(Owner, self).__init__(name=name) + self.first = self.track_layer(core.Dense( + 1, name="first_layer", use_bias=False)) + + def call(self, x): + return self.first(x) + + first_owner = Owner() + + class User(network.Network): + + def __init__(self, use_layer, name=None): + super(User, self).__init__(name=name) + self.first = self.track_layer(use_layer) + self.second = self.track_layer(core.Dense( + 1, name="second_layer", use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + class LikeUserButNotSharing(network.Network): + + def __init__(self, name=None): + super(LikeUserButNotSharing, self).__init__(name=name) + self.first = self.track_layer(core.Dense( + 1, name="first_layer", use_bias=False)) + self.second = self.track_layer(core.Dense( + 1, name="second_layer", use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + checkpoint_creator = LikeUserButNotSharing(name="checkpoint_creator") + one = constant_op.constant([[1.0]]) + checkpoint_creator(one) + self.assertEqual(2, len(checkpoint_creator.variables)) + self.evaluate(checkpoint_creator.variables[0].assign([[5.]])) + self.evaluate(checkpoint_creator.variables[1].assign([[6.]])) + # Re-map the variable names so that with default restore mapping we'll + # attempt to restore into the unbuilt Layer. + name_mapping = { + "checkpoint_creator/first_layer/kernel": "owner_1/first_layer/kernel", + "checkpoint_creator/second_layer/kernel": "second_layer/kernel", + } + save_path = checkpoint_creator.save( + self.get_temp_dir(), + map_func=lambda full_name: name_mapping[full_name]) + load_into = User(use_layer=first_owner.first) + load_into.restore(save_path) + self.assertEqual(0, len(first_owner.variables)) + self.assertAllEqual(self.evaluate(checkpoint_creator(one)), + self.evaluate(load_into(one))) + self.assertEqual(1, len(first_owner.variables)) + self.assertAllEqual([[5.]], self.evaluate(load_into.variables[0])) + self.assertAllEqual([[6.]], self.evaluate(load_into.variables[1])) + first_owner(one) + self.assertAllEqual([[5.]], self.evaluate(first_owner.variables[0])) + + # Try again with a garbage collected parent. + first_owner = Owner() + load_into = User(use_layer=first_owner.first) + del first_owner + gc.collect() + def _restore_map_func(original_name): + if original_name.startswith("owner_1"): + return original_name.replace("owner_1", "owner_2") + else: + return "user_2/" + original_name + with self.assertRaisesRegexp(ValueError, "garbage collected"): + load_into.restore(save_path, map_func=_restore_map_func) + + @test_util.run_in_graph_and_eager_modes() + def testRestoreIntoSubNetwork(self): + + class Parent(network.Network): + + def __init__(self, name=None): + super(Parent, self).__init__(name=name) + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.first(self.second(x)) + + one = constant_op.constant([[3.]]) + whole_model_saver = Parent() + whole_model_saver(one) + self.evaluate(whole_model_saver.variables[0].assign([[15.]])) + self.evaluate(whole_model_saver.variables[1].assign([[16.]])) + whole_model_checkpoint = whole_model_saver.save(self.get_temp_dir()) + + save_from = MyNetwork() + save_from(one) + self.evaluate(save_from.variables[0].assign([[5.]])) + checkpoint = save_from.save(self.get_temp_dir()) + save_into_parent = Parent() + save_into_parent.restore(whole_model_checkpoint) + save_into_parent.first.restore(checkpoint) + save_into_parent.first.restore(checkpoint) # deferred loading multiple + # times is fine + save_into_parent(one) # deferred loading + self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[0])) + self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1])) + + # Try again with the opposite ordering, and we should get different results + # (deferred restoration should happen the same way non-deferred happens, + # with later restorations overwriting older ones). + save_into_parent = Parent() + save_into_parent.first.restore(checkpoint) # deferred loading multiple + # times is fine + save_into_parent.restore(whole_model_checkpoint) + save_into_parent(one) # deferred loading + # We've overwritten the sub-Network restore. + self.assertAllEqual([[15.]], self.evaluate(save_into_parent.variables[0])) + self.assertAllEqual([[16.]], self.evaluate(save_into_parent.variables[1])) + + self.evaluate(save_into_parent.variables[0].assign([[3.]])) + self.evaluate(save_into_parent.variables[1].assign([[4.]])) + save_into_parent.second.restore(checkpoint) + self.assertAllEqual([[5.]], self.evaluate(save_into_parent.variables[1])) + with self.assertRaisesRegexp(errors_impl.NotFoundError, + "not found in checkpoint"): + # The checkpoint is incompatible. + save_into_parent.restore(checkpoint) + + @test_util.run_in_graph_and_eager_modes() + def testCustomMapCollisionErrors(self): + + class Parent(network.Network): + + def __init__(self, name=None): + super(Parent, self).__init__(name=name) + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.first(self.second(x)) + + make_checkpoint = Parent() + one = constant_op.constant([[1.]]) + make_checkpoint(one) + self.evaluate(make_checkpoint.variables[0].assign([[2.]])) + self.evaluate(make_checkpoint.variables[1].assign([[3.]])) + with self.assertRaisesRegexp( + ValueError, + "The map_func passed to Network.save for the Network 'parent_1' " + "resulted in two variables named 'foo'"): + make_checkpoint.save(self.get_temp_dir(), map_func=lambda n: "foo") + checkpoint = make_checkpoint.first.save( + self.get_temp_dir(), map_func=lambda n: "foo") + loader = Parent() + loader.restore(checkpoint, map_func=lambda n: "foo") + with self.assertRaisesRegexp( + ValueError, + ("The map_func passed to Network.restore for the Network" + " 'parent_2' resulted in two variables named 'foo'")): + loader(one) + loader = Parent() + loader(one) + with self.assertRaisesRegexp( + ValueError, + ("The map_func passed to Network.restore for the Network" + " 'parent_3' resulted in two variables named 'foo'")): + loader.restore(checkpoint, map_func=lambda n: "foo") + + @test_util.run_in_graph_and_eager_modes() + def testDefaultMapCollisionErrors(self): + + one = constant_op.constant([[1.]]) + first = core.Dense(1, name="dense_1", use_bias=False) + first(one) + + class Parent(network.Network): + + def __init__(self, name=None): + super(Parent, self).__init__(name=name) + self.first = self.track_layer(first) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.first(self.second(x)) + + make_checkpoint = Parent() + one = constant_op.constant([[1.]]) + make_checkpoint(one) + self.evaluate(make_checkpoint.variables[0].assign([[2.]])) + self.evaluate(make_checkpoint.variables[1].assign([[3.]])) + with self.assertRaisesRegexp( + ValueError, + ("The default checkpoint variable name mapping strategy for Network " + "'parent_1' resulted in a naming conflict.")): + make_checkpoint.save(self.get_temp_dir()) + + class Compatible(network.Network): + + def __init__(self, name=None): + super(Compatible, self).__init__(name=name) + self.first = self.track_layer(core.Dense(1, use_bias=False)) - def testNetworkAsAGraph(self): - self.skipTest("TODO(ashankar,josh11b): FIX THIS") - # Verify that we're using ResourceVariables + def call(self, x): + return self.first(x) + successful_checkpoint = Compatible() + successful_checkpoint(one) + self.evaluate(successful_checkpoint.variables[0].assign([[-1.]])) + checkpoint_path = successful_checkpoint.save(self.get_temp_dir()) + load_checkpoint = Parent() + load_checkpoint(one) + with self.assertRaisesRegexp( + ValueError, + ("The default checkpoint variable name mapping strategy for Network " + "'parent_2' resulted in a naming conflict.")): + load_checkpoint.restore(checkpoint_path) + + def testNoReferenceCyclesAfterCall(self): + + class ChildNetwork(network.Network): + + def __init__(self, name=None): + super(ChildNetwork, self).__init__(name=name) + + def call(self, x): + return x * 2. + + class ParentNetwork(network.Network): + + def __init__(self, name=None): + super(ParentNetwork, self).__init__(name=name) + self.l1 = self.track_layer(ChildNetwork()) + + def call(self, x): + return self.l1(x) + + one = constant_op.constant([[1.0]]) + gc.disable() + gc.collect() + previous_gc_debug_flags = gc.get_debug() + gc.set_debug(gc.DEBUG_SAVEALL) + preexisting = len(gc.garbage) + net = ParentNetwork() + net(one) + del net + gc.collect() + # There should be no additional garbage requiring collection. + self.assertEqual(preexisting, len(gc.garbage)) + gc.set_debug(previous_gc_debug_flags) + gc.enable() + + @test_util.run_in_graph_and_eager_modes() + def testAnonymousNoNameInitially(self): + net = MyNetwork() + with self.assertRaisesRegexp(ValueError, "does not yet have a final name"): + net.name # pylint: disable=pointless-statement + + @test_util.run_in_graph_and_eager_modes() + def testExplicitHasNameInitially(self): + net = MyNetwork(name="abcd") + self.assertEqual("abcd", net.name) + + @test_util.run_in_graph_and_eager_modes() + def testUsingResourceVariables(self): + net = MyNetwork() + net(constant_op.constant([[0.]])) + self.assertIsInstance(net.trainable_weights[0], + resource_variable_ops.ResourceVariable) + + @test_util.run_in_graph_and_eager_modes() + def testDuplicateNameError(self): + one = constant_op.constant([[1.]]) + net = MyNetwork(name="foo") + net(one) + with self.assertRaisesRegexp( + ValueError, "named 'foo' already exists"): + net1 = MyNetwork(name="foo") + net1(one) + + @test_util.run_in_graph_and_eager_modes() + def testWrappingInVariableScope(self): + with variable_scope.variable_scope("outside_scope"): + net = MyNetwork() + one = constant_op.constant([[1.]]) + with self.assertRaisesRegexp( + ValueError, + ("Creating Networks inside named variable_scopes is currently not " + "supported")): + net(one) + # Alternatively, we could re-name the Network to match the variable_scope: + # self.assertEqual("outside_scope/my_network_1", net.name) + # self.assertStartsWith( + # expected_start="outside_scope/my_network_1/dense/", + # actual=net.trainable_weights[0].name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerNamesRespected(self): + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer( + core.Dense(1, use_bias=False, name="explicit_name")) + + def call(self, x): + return self.first(x) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith(expected_start="parent_network_1/explicit_name/", + actual=net.trainable_weights[0].name) + self.assertEqual("explicit_name", net.first.name) + + @test_util.run_in_graph_and_eager_modes() + def testWrappingInAnonymousVariableScope(self): + # Named outside variable_scopes are not supported at the moment. However, + # blank-named top level variable scopes do not change variable names, and so + # can be used to set the properties of Network variables. + was_called = [False] + def _custom_getter(getter, *args, **kwargs): + was_called[0] = True + return getter(*args, **kwargs) + with variable_scope.variable_scope("", custom_getter=_custom_getter): + net = MyNetwork() + one = constant_op.constant([[1.]]) + net(one) + self.assertTrue(was_called[0]) + + @test_util.run_in_graph_and_eager_modes() + def testReasonableSlashError(self): + with self.assertRaisesRegexp( + ValueError, "not allowed in Network names"): + MyNetwork(name="slash/slash") + + @test_util.run_in_graph_and_eager_modes() + def testNoVariableScopeNames(self): + with self.assertRaisesRegexp( + ValueError, "VariableScopes are not valid Network names"): + with variable_scope.variable_scope("some_scope") as vs: + MyNetwork(name=vs) + + @test_util.run_in_graph_and_eager_modes() + def testVariableScopeNameCollision(self): + with variable_scope.variable_scope("abcd"): + pass + with self.assertRaisesRegexp( + ValueError, "or a variable_scope was created with this name"): + net = MyNetwork(name="abcd") + one = constant_op.constant([[1.]]) + net(one) + + @test_util.run_in_graph_and_eager_modes() def testNetworkVariablesDoNotInterfere(self): - self.skipTest("TODO: FIX THIS") + core.Dense(1, use_bias=True) # Should not interfere with naming. net1 = MyNetwork() net2 = MyNetwork() + one = constant_op.constant([[1.]]) + net1(one) + net2(one) + # Layer names typically are globally unique rather than being unique within + # the scope of their first use. However, within a Network they must be named + # locally so that previous Layer consutrciton does not interfere with + # variable naming (e.g. add a Layer construction before the Network, + # suddenly your previously saved checkpoint is incompatible). + self.assertEqual("dense_1", net1.l1.name) + self.assertEqual("dense_1", net2.l1.name) + self.evaluate(net1.trainable_weights[0].assign([[1.]])) + self.evaluate(net2.trainable_weights[0].assign([[2.]])) + self.assertEqual(2., self.evaluate(net2.trainable_weights[0])) + self.assertEqual(1., self.evaluate(net1.trainable_weights[0])) + self.assertStartsWith(expected_start="my_network_1/dense_1/", + actual=net1.trainable_weights[0].name) + self.assertStartsWith(expected_start="my_network_2/dense_1/", + actual=net2.trainable_weights[0].name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableAnonymous(self): + + # The case where no explicit names are specified. We make up unique names, + # and these should match the variable names. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense", + actual=net.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_1/my_network_1/dense", + actual=net.first.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_1/my_network_2/dense", + actual=net.trainable_weights[1].name) + self.assertStartsWith(expected_start="parent_network_1/my_network_2/dense", + actual=net.second.trainable_weights[0].name) + self.assertEqual("parent_network_1", net.name) + self.assertEqual("my_network_1", net.first.name) + self.assertEqual("my_network_2", net.second.name) - print(type(net1(one))) + net2 = ParentNetwork() net2(one) + self.assertStartsWith(expected_start="parent_network_2/my_network_1/dense", + actual=net2.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_2/my_network_1/dense", + actual=net2.first.trainable_weights[0].name) + self.assertStartsWith(expected_start="parent_network_2/my_network_2/dense", + actual=net2.trainable_weights[1].name) + self.assertStartsWith(expected_start="parent_network_2/my_network_2/dense", + actual=net2.second.trainable_weights[0].name) + self.assertEqual("parent_network_2", net2.name) + self.assertEqual("my_network_1", net2.first.name) + self.assertEqual("my_network_2", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicit(self): - net1.trainable_weights[0].assign(constant_op.constant([[1.]])) - net2.trainable_weights[0].assign(constant_op.constant([[2.]])) + # We have explicit network names and everything is globally unique. + class ParentNetwork(network.Network): - print("NET1") - print(net1.name) - print(net1.variables) - print(net1(one)) + def __init__(self): + super(ParentNetwork, self).__init__(name="unique_parent_name") + self.first = self.track_layer( + MyNetwork(name="first_unique_child_name")) + self.second = self.track_layer( + MyNetwork(name="second_unique_child_name")) - print("NET2") - print(net2.name) - print(net2.variables) - print(net2(one)) + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith( + expected_start="unique_parent_name/first_unique_child_name/dense", + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="unique_parent_name/second_unique_child_name/dense", + actual=net.trainable_weights[1].name) + self.assertEqual("unique_parent_name", net.name) + self.assertEqual("first_unique_child_name", net.first.name) + self.assertEqual("second_unique_child_name", net.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerNetworkNameInteractions(self): + + # Same base name as core.Dense; Networks and non-Network Layers with the + # same base name should use the same numbering system. + class Dense(network.Network): + + def __init__(self): + super(Dense, self).__init__() + self.first = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.first(x) + + class MixedLayerNetwork(network.Network): + + def __init__(self): + super(MixedLayerNetwork, self).__init__() + self.first = self.track_layer(core.Dense(1, use_bias=False)) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + self.third = self.track_layer(Dense()) + self.fourth = self.track_layer(core.Dense(1, use_bias=False)) + self.fifth = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.fifth(self.fourth(self.third(self.second(self.first(x))))) + + one = constant_op.constant([[1.]]) + net = MixedLayerNetwork() + net(one) + self.assertEqual("dense_1", net.first.name) + self.assertEqual("dense_2", net.second.name) + self.assertEqual("dense_3", net.third.name) + self.assertEqual("dense_4", net.fourth.name) + self.assertEqual("dense_5", net.fifth.name) + # Note that this is _not_ the default naming behavior for Layers. Layers + # which are added to Networks follow Network variable naming conventions + # (i.e. variable names = network name unless variable sharing). Nested + # Layers revert to Layer behavior. + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_1/", + actual=net.trainable_weights[0].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_2/", + actual=net.trainable_weights[1].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_3/", + actual=net.trainable_weights[2].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_4/", + actual=net.trainable_weights[3].name) + self.assertStartsWith(expected_start="mixed_layer_network_1/dense_5/", + actual=net.trainable_weights[4].name) + self.assertEqual("mixed_layer_network_1", net.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicitCollisions(self): + + # We have explicit network names and they are unique within the layer + # they're added to. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__(name="nonunique_name") + self.first = self.track_layer( + MyNetwork(name="nonunique_name")) + self.second = self.track_layer( + MyNetwork(name="second_unique_child_name")) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith( + expected_start="nonunique_name/nonunique_name/dense", + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="nonunique_name/second_unique_child_name/dense", + actual=net.trainable_weights[1].name) + self.assertEqual("nonunique_name", net.name) + self.assertEqual("nonunique_name", net.first.name) + self.assertEqual("second_unique_child_name", net.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicitWithAnonymousParent(self): + + # A parent network is instantiated multiple times with explicitly named + # children. We shouldn't throw any name errors. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer( + MyNetwork(name="first_unique_child_name")) + self.second = self.track_layer( + MyNetwork(name="second_unique_child_name")) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = ParentNetwork() + net(one) + self.assertStartsWith( + expected_start="parent_network_1/first_unique_child_name/dense_1/", + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="parent_network_1/second_unique_child_name/dense_1/", + actual=net.trainable_weights[1].name) + self.assertEqual("parent_network_1", net.name) + self.assertEqual("first_unique_child_name", net.first.name) + self.assertEqual("second_unique_child_name", net.second.name) + + net2 = ParentNetwork() + net2(one) + self.assertStartsWith( + expected_start="parent_network_2/first_unique_child_name/dense", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="parent_network_2/second_unique_child_name/dense", + actual=net2.trainable_weights[1].name) + self.assertEqual("parent_network_2", net2.name) + self.assertEqual("first_unique_child_name", net2.first.name) + self.assertEqual("second_unique_child_name", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testNestableExplicitSameLayerCollisions(self): + + # We have explicit network names and they are _not_ unique within the layer + # they're added to. Error. + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__(name="unique_parent_name") + self.first = self.track_layer(MyNetwork(name="nonunique_name")) + self.second = self.track_layer(MyNetwork(name="nonunique_name")) + + def call(self, x): + return self.second(self.first(x)) + + with self.assertRaisesRegexp(ValueError, "nonunique_name"): + ParentNetwork() + + @test_util.run_in_graph_and_eager_modes() + def testAnonymousVariableSharing(self): + + # Two "owned" Networks + class FirstParentNetwork(network.Network): + + def __init__(self): + super(FirstParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + one = constant_op.constant([[1.]]) + net = FirstParentNetwork() + net(one) + + # One Network shared with FirstParentNetwork, one owned Network. Same name, + # but this is OK because only one is owned. This name collision is + # avoidable; we could have looked at the base_name of the non-owned Network + # and incremented our naming based on that. + class SecondParentNetwork(network.Network): + + def __init__(self): + super(SecondParentNetwork, self).__init__() + self.first = self.track_layer(net.first) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + net2 = SecondParentNetwork() + net2(one) + + self.assertStartsWith( + expected_start="first_parent_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="second_parent_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[1].name) + self.assertEqual("second_parent_network_1", net2.name) + self.assertTrue(net2.first is net.first) + self.assertEqual("my_network_1", net2.first.name) + self.assertEqual("my_network_1", net2.second.name) + + # No name collision; the owned Network is added first and has a different + # name than the shared Network. + class ThirdParentNetwork(network.Network): + + def __init__(self): + super(ThirdParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(net.second) + + def call(self, x): + return self.second(self.first(x)) + + net3 = ThirdParentNetwork() + net3(one) + + self.assertStartsWith( + expected_start="third_parent_network_1/my_network_1/dense", + actual=net3.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_parent_network_1/my_network_2/dense", + actual=net3.trainable_weights[1].name) + self.assertEqual("third_parent_network_1", net3.name) + self.assertTrue(net3.second is net.second) + self.assertEqual("my_network_1", net3.first.name) + self.assertEqual("my_network_2", net3.second.name) + + # "Unavoidable" same-name Layer. The owned name is added first (fixed), then + # a shared Network is added with the same name. + class FourthParentNetwork(network.Network): + + def __init__(self): + super(FourthParentNetwork, self).__init__() + self.first = self.track_layer(MyNetwork()) + self.second = self.track_layer(net.first) + + def call(self, x): + return self.second(self.first(x)) + + net4 = FourthParentNetwork() + net4(one) + + self.assertStartsWith( + expected_start="fourth_parent_network_1/my_network_1/dense_1/", + actual=net4.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_parent_network_1/my_network_1/dense_1/", + actual=net4.trainable_weights[1].name) + self.assertEqual("fourth_parent_network_1", net4.name) + self.assertTrue(net4.second is net.first) + self.assertEqual("my_network_1", net4.first.name) + self.assertEqual("my_network_1", net4.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testRecursiveLayerRenaming(self): + core.Dense(1) # Under default Layer naming, would change subsequent names. + + class NetworkWithLayerChildren(network.Network): + + def __init__(self): + super(NetworkWithLayerChildren, self).__init__() + self.first = self.track_layer(core.Dense(1, use_bias=False)) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + class ParentNetwork(network.Network): + + def __init__(self): + super(ParentNetwork, self).__init__() + self.first = self.track_layer(NetworkWithLayerChildren()) + self.second = self.track_layer(NetworkWithLayerChildren()) + + def call(self, x): + return self.second(self.first(x)) + + net = ParentNetwork() + one = constant_op.constant([[1.]]) + net(one) + + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_1/" + "dense_1/"), + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_1/" + "dense_2/"), + actual=net.trainable_weights[1].name) + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_2/" + "dense_1/"), + actual=net.trainable_weights[2].name) + self.assertStartsWith( + expected_start=("parent_network_1/network_with_layer_children_2/" + "dense_2/"), + actual=net.trainable_weights[3].name) + self.assertEqual("parent_network_1", net.name) + self.assertEqual("network_with_layer_children_1", net.first.name) + self.assertEqual("network_with_layer_children_2", net.second.name) + self.assertEqual("dense_1", net.first.first.name) + self.assertEqual("dense_2", net.first.second.name) + self.assertEqual("dense_1", net.second.first.name) + self.assertEqual("dense_2", net.second.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testCallInDifferentOrderThanConstruct(self): + shared_network = MyNetwork() + + class FirstNetwork(network.Network): + + def __init__(self): + super(FirstNetwork, self).__init__() + self.first = self.track_layer(shared_network) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + class SecondNetwork(network.Network): + + def __init__(self): + super(SecondNetwork, self).__init__() + self.first = self.track_layer(shared_network) + self.second = self.track_layer(MyNetwork()) + + def call(self, x): + return self.second(self.first(x)) + + net1 = FirstNetwork() + net2 = SecondNetwork() + + one = constant_op.constant([[1.]]) + net2(one) + net1(one) + + self.assertStartsWith( + expected_start="first_network_1/my_network_1/dense_1/", + actual=net1.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_network_1/my_network_2/dense_1/", + actual=net1.trainable_weights[1].name) + self.assertStartsWith( + expected_start="first_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="second_network_1/my_network_1/dense_1/", + actual=net2.trainable_weights[1].name) + self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0]) + self.assertEqual("first_network_1", net1.name) + self.assertEqual("my_network_1", net1.first.name) + self.assertEqual("my_network_2", net1.second.name) + self.assertTrue(net2.first is net1.first) + self.assertEqual("my_network_1", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerCallInDifferentOrderThanConstruct(self): + # Same idea as testCallInDifferentOrderThanConstruct, but this time with a + # non-Network Layer shared between two Networks rather than a + # Network. Naming should follow the same rules. + shared_layer = core.Dense(1, use_bias=False) + + class FirstNetwork(network.Network): + + def __init__(self): + super(FirstNetwork, self).__init__() + self.first = self.track_layer(shared_layer) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + class SecondNetwork(network.Network): + + def __init__(self): + super(SecondNetwork, self).__init__() + self.first = self.track_layer(shared_layer) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + net1 = FirstNetwork() + net2 = SecondNetwork() + + one = constant_op.constant([[1.]]) + net2(one) + net1(one) + + self.assertStartsWith( + expected_start="first_network_1/dense_1/", + actual=net1.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_network_1/dense_2/", + actual=net1.trainable_weights[1].name) + self.assertStartsWith( + expected_start="first_network_1/dense_1/", + actual=net2.trainable_weights[0].name) + self.assertStartsWith( + expected_start="second_network_1/dense_1/", + actual=net2.trainable_weights[1].name) + self.assertTrue(net1.trainable_weights[0] is net2.trainable_weights[0]) + self.assertEqual("first_network_1", net1.name) + self.assertEqual("dense_1", net1.first.name) + self.assertEqual("dense_2", net1.second.name) + self.assertTrue(net2.first is net1.first) + self.assertEqual("dense_1", net2.second.name) + + @test_util.run_in_graph_and_eager_modes() + def testLayerAlreadyBuilt(self): + one = constant_op.constant([[1.]]) + core.Dense(1, use_bias=False) # pre-built layers use global naming + one = constant_op.constant([[1.]]) + core.Dense(1, use_bias=False)(one) + shared_layer = core.Dense(1, use_bias=False) + shared_layer(one) + + class FirstNetwork(network.Network): + + def __init__(self): + super(FirstNetwork, self).__init__() + self.first = self.track_layer(shared_layer) + self.second = self.track_layer(core.Dense(1, use_bias=False)) + + def call(self, x): + return self.second(self.first(x)) + + net = FirstNetwork() + net(one) + + self.assertStartsWith( + expected_start="dense_1/", # Pre-built layers have variable names which + # do not match their layer names. + actual=net.trainable_weights[0].name) + self.assertStartsWith( + expected_start="first_network_1/dense_1/", + actual=net.trainable_weights[1].name) + self.assertTrue( + net.trainable_weights[0] is shared_layer.trainable_weights[0]) + self.assertEqual("first_network_1", net.name) + self.assertEqual("dense_3", net.first.name) + self.assertEqual("dense_1", net.second.name) class SequentialTest(test.TestCase): @@ -94,7 +1020,7 @@ class SequentialTest(test.TestCase): # Add a second layer to the network. l2 = core.Dense(1, use_bias=False) - net.add_layer(l2) + net.add(l2) # Set the second layer's weights so it multiplies by 11 net(constant_op.constant([[2.0]])) # Create l2's variables @@ -102,6 +1028,48 @@ class SequentialTest(test.TestCase): l2.trainable_variables[0].assign([[11.0]]) self.assertEqual(231.0, net(constant_op.constant([[7.0]])).numpy()) + def testFunctions(self): + # Create a sequential network with one function. + net = network.Sequential([nn_ops.relu]) + two = constant_op.constant(2.0) + self.assertEqual(2.0, net(two).numpy()) + self.assertEqual(0.0, net(-two).numpy()) + # Add a second function. + net.add(math_ops.negative) + self.assertEqual(-2.0, net(two).numpy()) + + def testTrainingLayer(self): + net = network.Sequential([core.Dropout(0.99999)]) + two = constant_op.constant(2.0) + self.assertEqual(2.0, net(two).numpy()) + self.assertEqual(2.0, net(two, training=False).numpy()) + for _ in range(20): + with_dropout = net(two, training=True).numpy() + self.assertIn(with_dropout, [0.0, 2.0]) + if with_dropout == 0.0: + return + # Should only fail spuriously 1 in 10^100 runs. + self.fail("Didn't see dropout happen after 20 tries.") + + def testTrainingFunction(self): + # Output depends on value of "training". + def add_training(input_value, training=None): + if training is None: + return input_value + elif training: + return input_value + 1 + return input_value - 1 + + # Passing a "training" argument to double would cause an error. + def double(input_value): + return 2 * input_value + + net = network.Sequential([add_training, double]) + two = constant_op.constant(2) + self.assertEqual(4, net(two).numpy()) + self.assertEqual(2, net(two, training=False).numpy()) + self.assertEqual(6, net(two, training=True).numpy()) + if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/eager/python/saver.py b/tensorflow/contrib/eager/python/saver.py index d289b83f537acc76fefa3343115a76c13ba7451b..e0a20d2485e831b1841991596b91429c6eaa2854 100644 --- a/tensorflow/contrib/eager/python/saver.py +++ b/tensorflow/contrib/eager/python/saver.py @@ -19,31 +19,34 @@ from __future__ import print_function import contextlib +from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.training import adam as _adam from tensorflow.python.training import checkpoint_utils from tensorflow.python.training import saver as _saver def _init_from_checkpoint(self, *args, **kwargs): """Overrides default init by loading value from checkpoint.""" - self.old_init(*args, **kwargs) # pylint: disable=protected-access - if self._shared_name not in self.ckpt_var_cache: + self._old_init(*args, **kwargs) + ckpt_name = self._map_func(self._shared_name) + if ckpt_name not in self._ckpt_var_cache: raise errors.NotFoundError(None, None, - "%s not found in checkpoint" % self._shared_name) + "%s not found in checkpoint" % ckpt_name) - val = self.ckpt_var_cache[self._shared_name] + val = self._ckpt_var_cache.get(ckpt_name, None) if val is not None: - self.assign(self.ckpt_var_cache[self._shared_name]) + self.assign(val) # Avoid assigning for the second time. - self.ckpt_var_cache[self._shared_name] = None + self._ckpt_var_cache[ckpt_name] = None # pylint: enable=protected-access @contextlib.contextmanager -def restore_variables_on_create(save_path): +def restore_variables_on_create(save_path, map_func=None): """ContextManager that restores variables on creation. When save_path is None (e.g. No checkpoint), does nothing. @@ -58,26 +61,45 @@ def restore_variables_on_create(save_path): Args: save_path: The checkpoint file prefix. + map_func: A function that given the variable name as argument + and returns a variable name in checkpoint for restore. If + None, use the variable with the same name in checkpoint to restore. + It's an error that the mapped variable name doesn't exist in + checkpoint. Yields: Nothing. Raises: NotFoundError: If the variable is not found in checkpoint. + ValueError: If not used in eager mode or map_func is not callable. """ + if context.in_graph_mode(): + raise ValueError( + "Currently, restore_variables_on_create can only be used with " + "eager execution enabled.") if save_path: + if map_func is None: + map_func_wrapper = lambda self, x: x + else: + if not callable(map_func): + raise ValueError("map_func must be callaled.") + map_func_wrapper = lambda self, x: map_func(x) + ckpt_var_cache = dict() reader = checkpoint_utils.load_checkpoint(save_path) for k, _ in checkpoint_utils.list_variables(save_path): ckpt_var_cache[k] = reader.get_tensor(k) - old_init = getattr( - resource_variable_ops.ResourceVariable, "_init_from_args", None) + old_init = getattr(resource_variable_ops.ResourceVariable, + "_init_from_args", None) assert old_init, "ResourceVariable misses _init_from_args method." setattr(resource_variable_ops.ResourceVariable, "_init_from_args", _init_from_checkpoint) - setattr(resource_variable_ops.ResourceVariable, "old_init", old_init) - setattr(resource_variable_ops.ResourceVariable, "ckpt_var_cache", + setattr(resource_variable_ops.ResourceVariable, "_old_init", old_init) + setattr(resource_variable_ops.ResourceVariable, "_map_func", + map_func_wrapper) + setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", ckpt_var_cache) try: yield @@ -87,43 +109,82 @@ def restore_variables_on_create(save_path): if save_path: setattr(resource_variable_ops.ResourceVariable, "_init_from_args", old_init) - setattr(resource_variable_ops.ResourceVariable, "old_init", None) - setattr(resource_variable_ops.ResourceVariable, "ckpt_var_cache", None) + setattr(resource_variable_ops.ResourceVariable, "_old_init", None) + setattr(resource_variable_ops.ResourceVariable, "_map_func", None) + setattr(resource_variable_ops.ResourceVariable, "_ckpt_var_cache", None) class Saver(object): - """A simple tf.train.Saver adapter for eager mode. - - save and restore API are similar to the tf.train.Saver, except that - session is not needed. - - Args: - var_list: A list of variables. + """A tf.train.Saver adapter for use when eager execution is enabled. """ def __init__(self, var_list): + """A tf.train.Saver adapter for use when eager execution is enabled. + + The API, and on-disk format, mimic tf.train.Saver except that no + Session is needed. + + Args: + var_list: The list of variables that will be saved and restored. Either a + list of `tfe.Variable` objects, or a dictionary mapping names to + `tfe.Variable` objects. + + Raises: + RuntimeError: if invoked when eager execution has not been enabled. + """ + if context.in_graph_mode(): + raise RuntimeError("tfe.Saver can only be used when eager " + "execution is enabled. Use tf.train.Saver when " + "building graphs.") self._saver = _saver.Saver(var_list=var_list) - def save(self, save_path, global_step=None): + def save(self, file_prefix, global_step=None): """Saves variables. Args: - save_path: See save method in tf.train.Saver. - global_step: See save method in tf.train.Saver. + file_prefix: Path prefix of files created for the checkpoint. + global_step: If provided the global step number is appended to file_prefix + to create the checkpoint filename. The optional argument can be a + Tensor, a Variable, or an integer. Returns: - See save method in tf.train.Saver. + A string: prefix of filenames created for the checkpoint. This may be + an extension of file_prefix that is suitable to pass as an argument + to a subsequent call to `restore()`. """ with ops.device("/device:CPU:0"): - return self._saver.save(None, save_path, write_meta_graph=False, - global_step=global_step) + return self._saver.save( + None, file_prefix, write_meta_graph=False, global_step=global_step) - def restore(self, save_path): + def restore(self, file_prefix): """Restores previously saved variables. Args: - save_path: See restore method in tf.train.Saver. + file_prefix: Path prefix where parameters were previously saved. + Typically obtained from a previous `save()` call, or from + @{tf.train.latest_checkpoint}. """ with ops.device("/device:CPU:0"): - self._saver.restore(None, save_path) + self._saver.restore(None, file_prefix) + + +def get_optimizer_variables(optimizer): + """Returns a list of variables for the given `tf.train.Optimizer`. + Args: + optimizer: An instance of `tf.train.Optimizer` which has created variables + (typically after a call to `Optimizer.minimize`). + Returns: + A list of variables which have been created by the `Optimizer`. Currently + returns all variables even if they were not created in the default graph, + but this behavior may change. + """ + variables = [] + # pylint: disable=protected-access + for _, variable_dict in optimizer._slots.items(): + for _, slot_for_variable in variable_dict.items(): + variables.append(slot_for_variable) + if isinstance(optimizer, _adam.AdamOptimizer): + variables.append(optimizer._beta1_power) + variables.append(optimizer._beta2_power) + return variables diff --git a/tensorflow/contrib/eager/python/saver_test.py b/tensorflow/contrib/eager/python/saver_test.py index 29af2b531f4dee7f46c1538ff23409ece5785ceb..abc7e3690c76c4446bce6b945325f1ca15ef1c8b 100644 --- a/tensorflow/contrib/eager/python/saver_test.py +++ b/tensorflow/contrib/eager/python/saver_test.py @@ -22,6 +22,7 @@ import os from tensorflow.contrib.eager.python import saver as _saver from tensorflow.python.eager import context from tensorflow.python.eager import graph_callable +from tensorflow.python.eager import test from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops @@ -29,7 +30,10 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope -from tensorflow.python.platform import test +from tensorflow.python.training import adam +from tensorflow.python.training import gradient_descent +from tensorflow.python.training import momentum +from tensorflow.python.training import rmsprop class SaverTest(test.TestCase): @@ -38,7 +42,7 @@ class SaverTest(test.TestCase): return '/device:GPU:0' if context.num_gpus() else '/device:CPU:0' def testBasics(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') def model(): return array_ops.constant(2.0) * v1 @@ -54,8 +58,76 @@ class SaverTest(test.TestCase): saver.restore(ckpt_prefix) self.assertEqual(v1.read_value().numpy(), 1.0) + def testSameNameNoClobbering(self): + with ops.device(self._dev()): + # Note that this test purposefully uses Graphs rather than + # IsolateTest. Users are more likely to accidentally create the same + # variable name this way. + first_graph = ops.Graph() + with first_graph.as_default(): + v1_first_graph = resource_variable_ops.ResourceVariable(1.0, name='v1') + with ops.Graph().as_default(): + v1_second_graph = resource_variable_ops.ResourceVariable(2.0, name='v1') + saver = _saver.Saver([v1_first_graph, v1_second_graph]) + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + with self.assertRaisesRegexp(ValueError, 'v1'): + saver.save(ckpt_prefix) + + def testDifferentGraphError(self): + with ops.device(self._dev()): + with ops.Graph().as_default(): + v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') + with ops.Graph().as_default(): + saver = _saver.Saver([v1]) + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + with self.assertRaisesRegexp(ValueError, 'Graph'): + saver.save(ckpt_prefix) + + def testSameObjectOK(self): + with ops.device(self._dev()): + v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') + # While different objects with the same shared_name are not good, passing + # in the same object multiple times is fine. + saver = _saver.Saver([v1, v1]) + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + saver.save(ckpt_prefix) + + def testSaveByDict(self): + with ops.device(self._dev()): + v1 = resource_variable_ops.ResourceVariable(1.0, name='v1') + v2 = resource_variable_ops.ResourceVariable(1.0, name='v2') + def model(): + return array_ops.constant(2.0) * v1 * v2 + + ckpt_prefix = os.path.join(test.get_temp_dir(), 'ckpt') + + # Save the variables under different names. + _ = model() + saver = _saver.Saver({'ckpt/v1': v1, 'ckpt/v2': v2}) + saver.save(ckpt_prefix) + v1.assign(2.0) + v2.assign(2.0) + self.assertEqual(v1.read_value().numpy(), 2.0) + self.assertEqual(v2.read_value().numpy(), 2.0) + # Can still restore it. + saver.restore(ckpt_prefix) + self.assertEqual(v1.read_value().numpy(), 1.0) + self.assertEqual(v1.read_value().numpy(), 1.0) + # However, cannot restore it with default name. + with self.assertRaisesOpError('not found in checkpoint'): + saver = _saver.Saver([v1, v2]).restore(ckpt_prefix) + + # Can specify which variable in ckpt to restore to which variable. + def map_func(x): + return {'v3': 'ckpt/v1', 'v4': 'ckpt/v2'}.get(x, x) + with _saver.restore_variables_on_create(ckpt_prefix, map_func): + v3 = resource_variable_ops.ResourceVariable(2.0, name='v3') + v4 = resource_variable_ops.ResourceVariable(2.0, name='v4') + self.assertEqual(v3.read_value().numpy(), 1.0) + self.assertEqual(v4.read_value().numpy(), 1.0) + def testRestoreOnCreate(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): def model(init_val): v1 = resource_variable_ops.ResourceVariable(init_val, name='v1') return array_ops.constant(1.0) * v1, v1 @@ -71,12 +143,9 @@ class SaverTest(test.TestCase): # Value is from checkpoint, but not from argument. ret, _ = model(2.0) self.assertEqual(ret.numpy(), 1.0) - # Create it a second time won't re-assign the checkpoint value. - v1_2 = resource_variable_ops.ResourceVariable(3.0, name='v1') - self.assertEqual(v1_2.read_value().numpy(), 3.0) def testRestoreNotFound(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): def model(v): return array_ops.constant(1.0) * v @@ -92,7 +161,7 @@ class SaverTest(test.TestCase): _ = model(resource_variable_ops.ResourceVariable(1.0, name='v2')) def testSaveRestoreGraphCallable(self): - with context.eager_mode(), ops.device(self._dev()): + with ops.device(self._dev()): @graph_callable.graph_callable( [graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32)]) def model(x): @@ -139,5 +208,42 @@ class SaverTest(test.TestCase): 3, model2(array_ops.constant(2, dtype=dtypes.float32)).numpy()) +class GetOptimizerTests(test.TestCase): + + def _optimizer_test_template(self, optimizer): + """Checks save and restore. Returns the optimizer variables.""" + v = resource_variable_ops.ResourceVariable([[2., 3.]], name='v') + loss_fn = lambda: v[0, 0] ** 2 + v[0, 1] ** 2 + optimizer.minimize(loss_fn) + optimizer_variables = _saver.get_optimizer_variables(optimizer) + saver = _saver.Saver(optimizer_variables + [v]) + checkpoint_path = saver.save(self.get_temp_dir()) + optimizer.minimize(loss_fn) + after_first_minimize = v.numpy() + # After we restore, the next step should be exactly the same as the one we + # just did. + saver.restore(checkpoint_path) + optimizer.minimize(loss_fn) + self.assertAllEqual(after_first_minimize, v.numpy()) + return optimizer_variables + + def testAdam(self): + optimizer = adam.AdamOptimizer(0.1) + self._optimizer_test_template(optimizer) + + def testGradientDescent(self): + optimizer = gradient_descent.GradientDescentOptimizer(0.02) + self.assertEqual(0, len(self._optimizer_test_template(optimizer))) + + def testMomentum(self): + optimizer = momentum.MomentumOptimizer( + learning_rate=0.03, + momentum=0.5) + self._optimizer_test_template(optimizer) + + def testRMSProp(self): + optimizer = rmsprop.RMSPropOptimizer(0.01) + self._optimizer_test_template(optimizer) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/eager/python/summary_writer.py b/tensorflow/contrib/eager/python/summary_writer.py index 39993558e33d9f88c9f642db2273fb81fd7be9e9..5d8c41b545b3c9fd03af85f302ba05a394f085a4 100644 --- a/tensorflow/contrib/eager/python/summary_writer.py +++ b/tensorflow/contrib/eager/python/summary_writer.py @@ -32,9 +32,9 @@ from tensorflow.python.ops import summary_op_util from tensorflow.python.ops import variable_scope -def _maybe_as_cpu_tensor(v): +def _maybe_cpu(v): if isinstance(v, (ops.EagerTensor, ops.Tensor)): - return v.as_cpu_tensor() + return v.cpu() else: return v @@ -114,11 +114,9 @@ class SummaryWriter(object): self._resource = gen_summary_ops.summary_writer(shared_name=self._name) gen_summary_ops.create_summary_file_writer( self._resource, logdir, max_queue, flush_secs, filename_suffix) - - def __del__(self): - if self._resource: - resource_variable_ops.destroy_resource_op(self._resource) - self._resource = None + # Delete the resource when this object is deleted + self._resource_deleter = resource_variable_ops.EagerResourceDeleter( + handle=self._resource, handle_device=self._CPU_DEVICE) def step(self): """Increment the global step counter of this SummaryWriter instance.""" @@ -161,9 +159,9 @@ class SummaryWriter(object): gen_summary_ops.write_summary( self._resource, self._update_global_step_tensor(), - _maybe_as_cpu_tensor(tensor), + _maybe_cpu(tensor), tag, - _maybe_as_cpu_tensor(metadata), + _maybe_cpu(metadata), name=scope) def scalar(self, name, tensor, family=None): @@ -185,7 +183,7 @@ class SummaryWriter(object): name, family, values=[tensor]) as (tag, scope): gen_summary_ops.write_scalar_summary( self._resource, self._update_global_step_tensor(), - tag, _maybe_as_cpu_tensor(tensor), name=scope) + tag, _maybe_cpu(tensor), name=scope) def histogram(self, name, tensor, family=None): """Write a histogram summary. @@ -203,7 +201,7 @@ class SummaryWriter(object): name, family, values=[tensor]) as (tag, scope): gen_summary_ops.write_histogram_summary( self._resource, self._update_global_step_tensor(), - tag, _maybe_as_cpu_tensor(tensor), name=scope) + tag, _maybe_cpu(tensor), name=scope) def image(self, name, tensor, bad_color=None, max_images=3, family=None): """Write an image summary.""" @@ -214,7 +212,7 @@ class SummaryWriter(object): name, family, values=[tensor]) as (tag, scope): gen_summary_ops.write_image_summary( self._resource, self._update_global_step_tensor(), - tag, _maybe_as_cpu_tensor(tensor), bad_color_, max_images, + tag, _maybe_cpu(tensor), bad_color_, max_images, name=scope) def audio(self, name, tensor, sample_rate, max_outputs, family=None): @@ -238,7 +236,7 @@ class SummaryWriter(object): gen_summary_ops.write_audio_summary( self._resource, self._update_global_step_tensor(), tag, - _maybe_as_cpu_tensor(tensor), - sample_rate=_maybe_as_cpu_tensor(sample_rate), + _maybe_cpu(tensor), + sample_rate=_maybe_cpu(sample_rate), max_outputs=max_outputs, name=scope) diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py index 1acb1ba1b8c2aa2af0f7f24bd37b5afea09fe74f..b6c687c82946ec62ccb90165791587dc335f13c7 100644 --- a/tensorflow/contrib/eager/python/tfe.py +++ b/tensorflow/contrib/eager/python/tfe.py @@ -18,6 +18,8 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice. To use, at program startup, call `tfe.enable_eager_execution()`. +@@metrics + @@list_devices @@num_gpus @@ -26,6 +28,7 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@implicit_value_and_gradients @@gradients_function @@value_and_gradients_function +@@GradientTape @@enable_tracing @@flush_trace @@ -43,15 +46,22 @@ To use, at program startup, call `tfe.enable_eager_execution()`. @@seterr @@Iterator +@@Network @@Saver -@@SummaryWriter @@restore_variables_on_create @@Variable +@@get_optimizer_variables +@@EagerVariableStore @@in_eager_mode @@in_graph_mode +@@IsolateTest @@run_test_in_graph_and_eager_modes + +@@DEVICE_PLACEMENT_EXPLICIT +@@DEVICE_PLACEMENT_WARN +@@DEVICE_PLACEMENT_SILENT """ from __future__ import absolute_import @@ -61,18 +71,21 @@ from __future__ import print_function # pylint:disable=g-bad-import-order,g-import-not-at-top,unused-import # +from tensorflow.contrib.eager.python import metrics from tensorflow.contrib.eager.python.datasets import Iterator +from tensorflow.contrib.eager.python.network import Network +from tensorflow.contrib.eager.python.saver import get_optimizer_variables from tensorflow.contrib.eager.python.saver import restore_variables_on_create from tensorflow.contrib.eager.python.saver import Saver -from tensorflow.contrib.eager.python.summary_writer import SummaryWriter from tensorflow.python.eager import backprop from tensorflow.python.eager import function -from tensorflow.python.eager.context import enable_eager_execution +from tensorflow.python.eager.context import DEVICE_PLACEMENT_EXPLICIT +from tensorflow.python.eager.context import DEVICE_PLACEMENT_WARN +from tensorflow.python.eager.context import DEVICE_PLACEMENT_SILENT from tensorflow.python.eager.context import in_eager_mode from tensorflow.python.eager.context import in_graph_mode from tensorflow.python.eager.context import list_devices from tensorflow.python.eager.context import num_gpus -from tensorflow.python.eager.context import run from tensorflow.python.eager.core import enable_tracing from tensorflow.python.eager.custom_gradient import custom_gradient from tensorflow.python.eager.execution_callbacks import add_execution_callback @@ -81,8 +94,12 @@ from tensorflow.python.eager.execution_callbacks import inf_callback from tensorflow.python.eager.execution_callbacks import inf_nan_callback from tensorflow.python.eager.execution_callbacks import nan_callback from tensorflow.python.eager.execution_callbacks import seterr +from tensorflow.python.framework.ops import enable_eager_execution +from tensorflow.python.framework.ops import eager_run as run +from tensorflow.python.framework.test_util import IsolateTest from tensorflow.python.framework.test_util import run_in_graph_and_eager_modes as run_test_in_graph_and_eager_modes from tensorflow.python.ops.resource_variable_ops import ResourceVariable as Variable +from tensorflow.python.ops.variable_scope import EagerVariableStore from tensorflow.python.util.all_util import remove_undocumented defun = function.defun @@ -90,5 +107,6 @@ implicit_gradients = backprop.implicit_grad implicit_value_and_gradients = backprop.implicit_val_and_grad gradients_function = backprop.gradients_function value_and_gradients_function = backprop.val_and_grad_function +GradientTape = backprop.GradientTape # pylint: disable=invalid-name remove_undocumented(__name__) diff --git a/tensorflow/contrib/eager/python/tfe_test.py b/tensorflow/contrib/eager/python/tfe_test.py index 3d57a98a2ee068281b0934484994e113989e75ce..0dedb2fd7c0905801cd87c239ff2ee09eecb6080 100644 --- a/tensorflow/contrib/eager/python/tfe_test.py +++ b/tensorflow/contrib/eager/python/tfe_test.py @@ -17,6 +17,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import tempfile + from tensorflow.contrib.eager.python import tfe from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors @@ -24,7 +26,11 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import numerics +from tensorflow.python.ops import variables from tensorflow.python.platform import test +from tensorflow.python.summary import summary +from tensorflow.python.summary.writer import writer class TFETest(test_util.TensorFlowTestCase): @@ -39,6 +45,11 @@ class TFETest(test_util.TensorFlowTestCase): r'indices = 7 is not in \[0, 3\)'): array_ops.gather([0, 1, 2], 7) + def testVariableError(self): + with self.assertRaisesRegexp( + RuntimeError, r'Variable not supported in Eager mode'): + variables.Variable(initial_value=1.0) + def testGradients(self): def square(x): @@ -75,7 +86,7 @@ class TFETest(test_util.TensorFlowTestCase): self.skipTest('No GPUs available') # tf.Tensor.as_gpu_device() moves a tensor to GPU. - x = constant_op.constant([[1., 2.], [3., 4.]]).as_gpu_tensor() + x = constant_op.constant([[1., 2.], [3., 4.]]).gpu() # Alternatively, tf.device() as a context manager places tensors and # operations. with ops.device('gpu:0'): @@ -85,7 +96,7 @@ class TFETest(test_util.TensorFlowTestCase): reduction_indices = range(x.shape.ndims) m = math_ops.reduce_mean(x, reduction_indices) # m is on GPU, bring it back to CPU and compare. - self.assertEqual(3.5, m.as_cpu_tensor().numpy()) + self.assertEqual(3.5, m.cpu().numpy()) def testListDevices(self): # Expect at least one device. @@ -95,12 +106,33 @@ class TFETest(test_util.TensorFlowTestCase): devices = tfe.list_devices() self.assertEqual(len(devices) - 1, tfe.num_gpus()) - def testCallingEnableEagerExecutionMoreThanOnce(self): - # Note that eager.test.main() has already invoked enable_eager_exceution(). + def testAddCheckNumericsOpsRaisesError(self): + with self.assertRaisesRegexp( + RuntimeError, + r'add_check_numerics_ops\(\) is not compatible with eager execution'): + numerics.add_check_numerics_ops() + + def testClassicSummaryOpsErrorOut(self): + x = constant_op.constant(42) + x_summary = summary.scalar('x', x) + y = constant_op.constant([1, 3, 3, 7]) + y_summary = summary.histogram('hist', y) + + with self.assertRaisesRegexp( + RuntimeError, + r'Merging tf\.summary\.\* ops is not compatible with eager execution'): + summary.merge([x_summary, y_summary]) + + with self.assertRaisesRegexp( + RuntimeError, + r'Merging tf\.summary\.\* ops is not compatible with eager execution'): + summary.merge_all() + + def testClassicSummaryFileWriterErrorsOut(self): with self.assertRaisesRegexp( - ValueError, r'Do not call tfe\.%s more than once in the same process' % - tfe.enable_eager_execution.__name__): - tfe.enable_eager_execution() + RuntimeError, + r'tf\.summary\.FileWriter is not compatible with eager execution'): + writer.FileWriter(tempfile.mkdtemp()) if __name__ == '__main__': diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD index 4dd9f19ec3123112ac2dd3a6f2db0da90492a234..a0f83ac10555913b5be177f0f2b00b2b0e30494a 100644 --- a/tensorflow/contrib/estimator/BUILD +++ b/tensorflow/contrib/estimator/BUILD @@ -82,7 +82,6 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", - "//tensorflow/python:util", "//tensorflow/python/estimator", "//tensorflow/python/estimator:model_fn", "//tensorflow/python/estimator:util", @@ -134,7 +133,9 @@ py_library( "//tensorflow/python/estimator:metric_keys", "//tensorflow/python/estimator:model_fn", "//tensorflow/python/estimator:prediction_keys", + "//tensorflow/python/estimator:util", "//tensorflow/python/ops/losses", + "//tensorflow/python/saved_model:signature_constants", ], ) @@ -153,6 +154,7 @@ py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", "//tensorflow/python:training", @@ -187,7 +189,8 @@ py_test( deps = [ ":logit_fns", "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_ops", + "//tensorflow/python:constant_op", + "//tensorflow/python:session", "//tensorflow/python/estimator:model_fn", ], ) diff --git a/tensorflow/contrib/estimator/python/estimator/extenders.py b/tensorflow/contrib/estimator/python/estimator/extenders.py index 3e5eb3390f62141a82b51011d278d995b488b5e7..29c3c7358534f6e8ebbd31cbfcd7e34086d9b506 100644 --- a/tensorflow/contrib/estimator/python/estimator/extenders.py +++ b/tensorflow/contrib/estimator/python/estimator/extenders.py @@ -27,7 +27,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib from tensorflow.python.ops import clip_ops from tensorflow.python.training import optimizer as optimizer_lib -from tensorflow.python.util import tf_inspect + _VALID_METRIC_FN_ARGS = set(['features', 'labels', 'predictions', 'config']) @@ -317,9 +317,6 @@ class _TransformGradients(optimizer_lib.Optimizer): def _verify_metric_fn_args(metric_fn): args = set(estimator_util.fn_args(metric_fn)) - if tf_inspect.ismethod(metric_fn): - if 'self' in args: - args.remove('self') invalid_args = list(args - _VALID_METRIC_FN_ARGS) if invalid_args: raise ValueError('metric_fn (%s) has following not expected args: %s' % diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py index e7fe454fbfecb3a4f14a8575f7cb80e21ac2e4ff..189f098005b8926bfb30b723cc989cb854a5d77e 100644 --- a/tensorflow/contrib/estimator/python/estimator/head.py +++ b/tensorflow/contrib/estimator/python/estimator/head.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.estimator import model_fn +from tensorflow.python.estimator import util from tensorflow.python.estimator.canned import head as head_lib from tensorflow.python.estimator.canned import metric_keys from tensorflow.python.estimator.canned import prediction_keys @@ -33,8 +34,11 @@ from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics as metrics_lib from tensorflow.python.ops import sparse_ops from tensorflow.python.ops.losses import losses +from tensorflow.python.saved_model import signature_constants from tensorflow.python.summary import summary +_DEFAULT_SERVING_KEY = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY + def multi_class_head(n_classes, weight_column=None, @@ -59,7 +63,7 @@ def multi_class_head(n_classes, `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for multi class classification. @@ -98,7 +102,7 @@ def binary_classification_head( `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for binary classification. @@ -129,7 +133,7 @@ def regression_head(weight_column=None, of the last dimension of the labels `Tensor` (typically, this has shape `[batch_size, label_dimension]`). name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for linear regression. @@ -144,6 +148,7 @@ def multi_label_head(n_classes, weight_column=None, thresholds=None, label_vocabulary=None, + loss_fn=None, name=None): """Creates a `_Head` for multi-label classification. @@ -155,6 +160,12 @@ def multi_label_head(n_classes, multi-hot tensor of shape `[batch_size, n_classes]`, or as an integer `SparseTensor` of class indices. + Also supports custom `loss_fn`. `loss_fn` takes `(labels, logits)` or + `(labels, logits, features)` as arguments and returns unreduced loss with + shape `[batch_size, 1]`. `loss_fn` must support indicator `labels` with shape + `[batch_size, n_classes]`. Namely, the head applies `label_vocabulary` to the + input labels before passing them to `loss_fn`. + Args: n_classes: Number of classes, must be greater than 1 (for 1 class, use `binary_classification_head`). @@ -171,8 +182,9 @@ def multi_label_head(n_classes, [0, n_classes) or multi-hot Tensor. If given, labels must be SparseTensor string type and have any value in `label_vocabulary`. Also there will be errors if vocabulary is not provided and labels are string. + loss_fn: Optional loss function. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for multi-label classification. @@ -198,9 +210,11 @@ def multi_label_head(n_classes, raise ValueError( 'Length of label_vocabulary must be n_classes ({}). ' 'Given: {}'.format(n_classes, len(label_vocabulary))) + if loss_fn: + _validate_loss_fn_args(loss_fn) return _MultiLabelHead( n_classes=n_classes, weight_column=weight_column, thresholds=thresholds, - label_vocabulary=label_vocabulary, name=name) + label_vocabulary=label_vocabulary, loss_fn=loss_fn, name=name) class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access @@ -211,11 +225,13 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access weight_column=None, thresholds=None, label_vocabulary=None, + loss_fn=None, name=None): self._n_classes = n_classes self._weight_column = weight_column self._thresholds = thresholds self._label_vocabulary = label_vocabulary + self._loss_fn = loss_fn self._name = name @property @@ -260,11 +276,19 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access def create_loss(self, features, mode, logits, labels): """See `Head`.""" - del mode, features # Unused for this head. + del mode # Unused for this head. processed_labels = self._process_labels(labels) - unweighted_loss = losses.sigmoid_cross_entropy( - multi_class_labels=processed_labels, logits=logits, - reduction=losses.Reduction.NONE) + if self._loss_fn: + unweighted_loss = _call_loss_fn( + loss_fn=self._loss_fn, labels=processed_labels, logits=logits, + features=features) + else: + unweighted_loss = losses.sigmoid_cross_entropy( + multi_class_labels=processed_labels, logits=logits, + reduction=losses.Reduction.NONE) + # Averages loss over classes. + unweighted_loss = math_ops.reduce_mean( + unweighted_loss, axis=-1, keep_dims=True) return head_lib.LossAndLabels( unweighted_loss=unweighted_loss, processed_labels=processed_labels) @@ -272,7 +296,7 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): logits = head_lib._check_logits(logits, self.logits_dimension) # pylint:disable=protected-access # Predict. @@ -284,22 +308,25 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access pred_keys.PROBABILITIES: probabilities, } if mode == model_fn.ModeKeys.PREDICT: + classifier_output = head_lib._classification_output( # pylint:disable=protected-access + scores=probabilities, n_classes=self._n_classes, + label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, export_outputs={ - '': export_output.ClassificationOutput(scores=probabilities) + _DEFAULT_SERVING_KEY: classifier_output, + head_lib._CLASSIFY_SERVING_KEY: classifier_output, # pylint:disable=protected-access + head_lib._PREDICT_SERVING_KEY: ( # pylint:disable=protected-access + export_output.PredictOutput(predictions)) }) # Eval. unweighted_loss, processed_labels = self.create_loss( features=features, mode=mode, logits=logits, labels=labels) - # Averages loss over classes. - per_example_loss = math_ops.reduce_mean( - unweighted_loss, axis=-1, keep_dims=True) weights = head_lib._weights(features, self._weight_column) # pylint:disable=protected-access training_loss = losses.compute_weighted_loss( - per_example_loss, weights=weights, reduction=losses.Reduction.SUM) + unweighted_loss, weights=weights, reduction=losses.Reduction.SUM) if mode == model_fn.ModeKeys.EVAL: return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.EVAL, @@ -309,7 +336,7 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access labels=processed_labels, probabilities=probabilities, weights=weights, - per_example_loss=per_example_loss)) + unweighted_loss=unweighted_loss)) # Train. if train_op_fn is None: @@ -330,16 +357,16 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access loss=training_loss, train_op=train_op_fn(training_loss)) - def _eval_metric_ops(self, labels, probabilities, weights, per_example_loss): + def _eval_metric_ops(self, labels, probabilities, weights, unweighted_loss): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope( - None, 'metrics', [labels, probabilities, weights, per_example_loss]): + None, 'metrics', [labels, probabilities, weights, unweighted_loss]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( - per_example_loss, weights=weights, name=keys.LOSS_MEAN), + unweighted_loss, weights=weights, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc( labels=labels, predictions=probabilities, weights=weights, @@ -377,3 +404,53 @@ class _MultiLabelHead(head_lib._Head): # pylint:disable=protected-access threshold=threshold, name=recall_key)) return metric_ops + + +def _validate_loss_fn_args(loss_fn): + """Validates loss_fn arguments. + + Required arguments: labels, logits. + Optional arguments: features. + + Args: + loss_fn: The loss function. + Raises: + ValueError: If the signature is unexpected. + """ + loss_fn_args = util.fn_args(loss_fn) + for required_arg in ['labels', 'logits']: + if required_arg not in loss_fn_args: + raise ValueError( + 'loss_fn must contain argument: {}. ' + 'Given arguments: {}'.format(required_arg, loss_fn_args)) + invalid_args = list(set(loss_fn_args) - set(['labels', 'logits', 'features'])) + if invalid_args: + raise ValueError('loss_fn has unexpected args: {}'.format(invalid_args)) + + +def _call_loss_fn(loss_fn, labels, logits, features): + """Calls loss_fn and checks the returned shape. + + Args: + loss_fn: The loss function. + labels: Processed labels Tensor. + logits: Logits Tensor of shape [batch_size, logits_dimension]. + features: Features dict. + Returns: + Loss Tensor with shape [batch_size, 1]. + """ + loss_fn_args = util.fn_args(loss_fn) + kwargs = {} + if 'features' in loss_fn_args: + kwargs['features'] = features + unweighted_loss = loss_fn(labels=labels, logits=logits, **kwargs) + batch_size = array_ops.shape(logits)[0] + loss_shape = array_ops.shape(unweighted_loss) + check_shape_op = control_flow_ops.Assert( + math_ops.reduce_all(math_ops.equal(loss_shape, [batch_size, 1])), + data=[ + 'loss_fn must return Tensor of shape [batch_size, 1]. Given: ', + loss_shape]) + with ops.control_dependencies([check_shape_op]): + return array_ops.identity(unweighted_loss) + diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py index dcbe62b49730baf6d9a98f49e71e9877b185aabb..db7d96d508649f93c23b55504088551747f15a26 100644 --- a/tensorflow/contrib/estimator/python/estimator/head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/head_test.py @@ -33,6 +33,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops from tensorflow.python.ops import string_ops from tensorflow.python.platform import test from tensorflow.python.saved_model import signature_constants @@ -80,9 +81,13 @@ def _sigmoid(logits): def _sigmoid_cross_entropy(labels, logits): + """Returns sigmoid cross entropy averaged over classes.""" sigmoid_logits = _sigmoid(logits) - return (-labels * np.log(sigmoid_logits) - -(1 - labels) * np.log(1 - sigmoid_logits)) + unreduced_result = ( + -labels * np.log(sigmoid_logits) + -(1 - labels) * np.log(1 - sigmoid_logits)) + # Mean over classes + return np.mean(unreduced_result, axis=-1, keepdims=True) class MultiLabelHead(test.TestCase): @@ -127,6 +132,37 @@ class MultiLabelHead(test.TestCase): r'Length of label_vocabulary must be n_classes \(3\). Given: 2'): head_lib.multi_label_head(n_classes=3, label_vocabulary=['foo', 'bar']) + def test_loss_fn_arg_labels_missing(self): + def _loss_fn(logits): + del logits # Unused + with self.assertRaisesRegexp( + ValueError, + r'loss_fn must contain argument: labels\. ' + r'Given arguments: \(\'logits\',\)'): + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + + def test_loss_fn_arg_logits_missing(self): + def _loss_fn(labels): + del labels # unused + with self.assertRaisesRegexp( + ValueError, + r'loss_fn must contain argument: logits\. ' + r'Given arguments: \(\'labels\',\)'): + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + + def test_loss_fn_arg_features_ok(self): + def _loss_fn(labels, logits, features): + del labels, logits, features # Unused + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + + def test_loss_fn_arg_invalid(self): + def _loss_fn(labels, logits, name=None): + del labels, logits, name # Unused + with self.assertRaisesRegexp( + ValueError, + r'loss_fn has unexpected args: \[\'name\'\]'): + head_lib.multi_label_head(n_classes=3, loss_fn=_loss_fn) + def test_name(self): head = head_lib.multi_label_head(n_classes=4, name='foo') self.assertEqual('foo', head.name) @@ -139,6 +175,7 @@ class MultiLabelHead(test.TestCase): logits = np.array( [[0., 1., 2., -1.], [-1., -2., -3., 1.]], dtype=np.float32) expected_probabilities = _sigmoid(logits) + expected_export_classes = [[b'0', b'1', b'2', b'3']] * 2 spec = head.create_estimator_spec( features={'x': np.array(((42,),), dtype=np.int32)}, @@ -146,7 +183,8 @@ class MultiLabelHead(test.TestCase): logits=logits) self.assertItemsEqual( - ('', _DEFAULT_SERVING_KEY), spec.export_outputs.keys()) + (_DEFAULT_SERVING_KEY, 'predict', 'classification'), + spec.export_outputs.keys()) # Assert predictions and export_outputs. with self.test_session() as sess: @@ -162,6 +200,29 @@ class MultiLabelHead(test.TestCase): self.assertAllClose( expected_probabilities, sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].scores)) + self.assertAllEqual( + expected_export_classes, + sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].classes)) + + def test_predict_with_label_vocabulary(self): + n_classes = 4 + head = head_lib.multi_label_head( + n_classes, label_vocabulary=['foo', 'bar', 'foobar', 'barfoo']) + + logits = np.array( + [[0., 1., 2., -1.], [-1., -2., -3., 1.]], dtype=np.float32) + expected_export_classes = [[b'foo', b'bar', b'foobar', b'barfoo']] * 2 + + spec = head.create_estimator_spec( + features={'x': np.array(((42,),), dtype=np.int32)}, + mode=model_fn.ModeKeys.PREDICT, + logits=logits) + + with self.test_session() as sess: + _initialize_variables(self, spec.scaffold) + self.assertAllEqual( + expected_export_classes, + sess.run(spec.export_outputs[_DEFAULT_SERVING_KEY].classes)) def test_weight_should_not_impact_prediction(self): n_classes = 4 @@ -226,7 +287,7 @@ class MultiLabelHead(test.TestCase): # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits expected_unweighted_loss = np.array( - [[10., 10.], [15., 0.]], dtype=np.float32) + [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32) actual_unweighted_loss, _ = head.create_loss( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.EVAL, @@ -262,6 +323,54 @@ class MultiLabelHead(test.TestCase): actual_unweighted_loss.eval( {labels_placeholder: np.array([1, 1], dtype=np.int64)}) + def test_eval_create_loss_loss_fn(self): + """Tests head.create_loss for eval mode and custom loss_fn.""" + loss = np.array([[1.], [2.]], dtype=np.float32) + logits_input = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32) + labels_input = np.array([[1, 0], [1, 1]], dtype=np.int64) + def _loss_fn(labels, logits): + check_labels = control_flow_ops.Assert( + math_ops.reduce_all(math_ops.equal(labels, labels_input)), + data=[labels]) + check_logits = control_flow_ops.Assert( + math_ops.reduce_all(math_ops.equal(logits, logits_input)), + data=[logits]) + with ops.control_dependencies([check_labels, check_logits]): + return constant_op.constant(loss) + head = head_lib.multi_label_head(n_classes=2, loss_fn=_loss_fn) + + actual_unweighted_loss, _ = head.create_loss( + features={'x': np.array(((42,),), dtype=np.int32)}, + mode=model_fn.ModeKeys.EVAL, + logits=logits_input, + labels=labels_input) + with self.test_session(): + _initialize_variables(self, monitored_session.Scaffold()) + self.assertAllClose(loss, actual_unweighted_loss.eval()) + + def test_eval_create_loss_loss_fn_wrong_shape(self): + """Tests custom loss_fn that returns Tensor of unexpected shape.""" + loss = np.array([1., 2.], dtype=np.float32) + def _loss_fn(labels, logits): + del labels, logits # Unused + return constant_op.constant(loss) + head = head_lib.multi_label_head(n_classes=2, loss_fn=_loss_fn) + + logits = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32) + labels = np.array([[1, 0], [1, 1]], dtype=np.int64) + actual_unweighted_loss, _ = head.create_loss( + features={'x': np.array(((42,),), dtype=np.int32)}, + mode=model_fn.ModeKeys.EVAL, + logits=logits, + labels=labels) + with self.test_session(): + _initialize_variables(self, monitored_session.Scaffold()) + with self.assertRaisesRegexp( + errors.InvalidArgumentError, + r'loss_fn must return Tensor of shape \[batch_size, 1\]\. ' + r'Given: \] \[2\]'): + actual_unweighted_loss.eval() + def test_eval_labels_none(self): """Tests that error is raised when labels is None.""" head = head_lib.multi_label_head(n_classes=2) @@ -311,10 +420,8 @@ class MultiLabelHead(test.TestCase): labels = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. - expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes - ) + # Sum over examples. + expected_loss = np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) keys = metric_keys.MetricKeys expected_metrics = { # Average loss over examples. @@ -343,10 +450,9 @@ class MultiLabelHead(test.TestCase): labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) / - n_classes + np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) ) keys = metric_keys.MetricKeys expected_metrics = { @@ -377,10 +483,9 @@ class MultiLabelHead(test.TestCase): labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) / - n_classes + np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits)) ) keys = metric_keys.MetricKeys expected_metrics = { @@ -407,9 +512,9 @@ class MultiLabelHead(test.TestCase): labels = np.array([[1, 0], [1, 1]], dtype=np.int64) # loss = labels * -log(sigmoid(logits)) + # (1 - labels) * -log(1 - sigmoid(logits)) - # Average over classes, and sum over examples. + # Sum over examples. expected_loss = ( - np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) / n_classes + np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits)) ) keys = metric_keys.MetricKeys @@ -506,7 +611,7 @@ class MultiLabelHead(test.TestCase): # loss = labels * (logits < 0) * (-logits) + # (1 - labels) * (logits > 0) * logits expected_unweighted_loss = np.array( - [[10., 10.], [15., 0.]], dtype=np.float32) + [[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32) actual_unweighted_loss, _ = head.create_loss( features={'x': np.array(((42,),), dtype=np.int32)}, mode=model_fn.ModeKeys.TRAIN, diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head.py b/tensorflow/contrib/estimator/python/estimator/multi_head.py index e6340424f741cd0278dbdef41dd4395e98f23246..64b2a9dee83801b5d6d852a3485fc0cc81417ff0 100644 --- a/tensorflow/contrib/estimator/python/estimator/multi_head.py +++ b/tensorflow/contrib/estimator/python/estimator/multi_head.py @@ -236,7 +236,10 @@ class _MultiHead(head_lib._Head): # pylint:disable=protected-access for head, spec in zip(self._heads, all_estimator_spec): head_name = head.name for k, v in six.iteritems(spec.export_outputs): - key = '%s/%s' % (k, head_name) if k else head_name + if k == _DEFAULT_SERVING_KEY: + key = head_name + else: + key = '%s/%s' % (k, head_name) export_outputs[key] = v for k, v in six.iteritems(spec.predictions): predictions[(head_name, k)] = v diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py index e86cb2b96fe1c10352337367616a0ea2ff9132cc..48027035cecffc3ce8aacf8ae917f5eb9e9b2473 100644 --- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py +++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py @@ -126,8 +126,8 @@ class MultiHeadTest(test.TestCase): logits=logits) self.assertItemsEqual( - (_DEFAULT_SERVING_KEY, _DEFAULT_SERVING_KEY + '/head1', 'head1', - _DEFAULT_SERVING_KEY + '/head2', 'head2'), + (_DEFAULT_SERVING_KEY, 'head1', 'classification/head1', 'predict/head1', + 'head2', 'classification/head2', 'predict/head2'), spec.export_outputs.keys()) # Assert predictions and export_outputs. diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD index 44095bd00a7a098a8a89ba4d25c68a2484c00a6e..fe86a20ab1f69a0eaf9d7486142451dac6337274 100644 --- a/tensorflow/contrib/factorization/BUILD +++ b/tensorflow/contrib/factorization/BUILD @@ -50,15 +50,22 @@ tf_custom_op_py_library( "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:linalg_ops", + "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:metrics", "//tensorflow/python:nn", "//tensorflow/python:platform", "//tensorflow/python:random_ops", "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", "//tensorflow/python:summary", + "//tensorflow/python:training", "//tensorflow/python:util", + "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/estimator", + "//tensorflow/python/estimator:model_fn", "//third_party/py/numpy", ], ) @@ -133,12 +140,17 @@ tf_py_test( ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", "//third_party/py/numpy", "//tensorflow/contrib/learn", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:data_flow_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform", "//tensorflow/python:platform_test", + "//tensorflow/python:random_ops", + "//tensorflow/python:random_seed", + "//tensorflow/python:training", ], tags = [ "no_pip", # b/38283730 @@ -162,6 +174,7 @@ tf_py_test( "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform", "//tensorflow/python:platform_test", + "//tensorflow/python:random_seed", "//tensorflow/python:variables", ], tags = ["notsan"], # b/62863147 @@ -193,10 +206,13 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", "//tensorflow/python:platform_test", + "//tensorflow/python:sparse_tensor", ], ) @@ -220,6 +236,7 @@ py_test( "//tensorflow/python:platform_benchmark", "//tensorflow/python:random_ops", "//tensorflow/python:training", + "//tensorflow/python/estimator:run_config", "//third_party/py/numpy", ], ) @@ -233,13 +250,20 @@ tf_py_test( ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", ":factorization_ops_test_utils_py", "//third_party/py/numpy", + "//tensorflow/contrib/learn", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:platform_benchmark", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", + "//tensorflow/python:training", "//tensorflow/python:variables", ], tags = [ @@ -256,11 +280,13 @@ tf_py_test( additional_deps = [ ":factorization_py", ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", + ":gen_factorization_ops", "//third_party/py/numpy", "//tensorflow/python:client_testlib", "//tensorflow/python:framework", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:sparse_tensor", ], ) @@ -284,10 +310,15 @@ tf_py_test( ":gen_factorization_ops", ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO", "//third_party/py/numpy", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", "//tensorflow/python:platform_test", + "//tensorflow/python:sparse_tensor", ], ) diff --git a/tensorflow/contrib/factorization/g3doc/kmeans.md b/tensorflow/contrib/factorization/g3doc/kmeans.md index b55c9d09ad386b84623d3648c5be83cbba8bbff9..c1843f0bf0704503d43c28d186dc826f0677711f 100644 --- a/tensorflow/contrib/factorization/g3doc/kmeans.md +++ b/tensorflow/contrib/factorization/g3doc/kmeans.md @@ -24,7 +24,11 @@ the full-batch version. approach for computing the initial cluster assignments that is expensive but is typically less prone to getting stuck in bad local minima. -We provide distributed implementations of both full-batch and mini-batch -K-Means algorithm. Both K-Means++ and random initialization are supported. -The user can also choose between **Cosine** and **Squared Euclidean** distance -metrics. +**[k-MC2](https://www.aaai.org/ocs/index.php/AAAI/AAAI16/paper/view/12147/11759)** +provides a very fast seeding method that provides high quality centers +comparable to K-Means++ seeding. k-MC2 works particularly well if it is combined +with Mini-batch K-Means. + +We provide distributed implementations of both full-batch and mini-batch K-Means +algorithm. K-Means++, k-MC2 and random initialization are supported. The user +can also choose between **Cosine** and **Squared Euclidean** distance metrics. diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc index a2136c08bbc2e91f4587b1cdacbfe3b1d1073949..dd61f59585aee2e0245cfd6797b313b972c19bc5 100644 --- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc +++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc @@ -224,6 +224,58 @@ class KmeansPlusPlusInitializationOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("KmeansPlusPlusInitialization").Device(DEVICE_CPU), KmeansPlusPlusInitializationOp); +// Implementation of one single Markov Chain for the k-MC^2 algorithm +class KMC2ChainInitializationOp : public OpKernel { + public: + explicit KMC2ChainInitializationOp(OpKernelConstruction* context) + : OpKernel(context) { + OP_REQUIRES_OK(context, + context->MatchSignature({DT_FLOAT, DT_INT64}, {DT_INT64})); + } + + void Compute(OpKernelContext* context) override { + const Tensor& distances_tensor = context->input(0); + const Tensor& seed_tensor = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsVector(distances_tensor.shape()), + InvalidArgument("Input distances should be a vector.")); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(seed_tensor.shape()), + InvalidArgument("Input seed should be a scalar.")); + const int64 num_points = distances_tensor.dim_size(0); + const int64 seed = seed_tensor.scalar()(); + OP_REQUIRES(context, num_points > 0, + InvalidArgument("Expected distances_tensor.size() > 0.")); + + random::PhiloxRandom random(seed); + random::SimplePhilox rng(&random); + + auto distances = distances_tensor.flat(); + // Set the initial state of the Markov chain to be the first candidate. + int64 selected_index = 0; + float selected_distance = distances(selected_index); + // Build a Markov chain of length num_points. + for (int64 i = 1; i < num_points; ++i) { + const float candidate_distance = distances(i); + // Set the next state of the Markov chain to be the candidate with + // probability min(1, candidate_distance/selected_distance). + if (candidate_distance > rng.RandFloat() * selected_distance) { + selected_index = i; + selected_distance = candidate_distance; + } + } + + Tensor* output_sampled_index_tensor; + OP_REQUIRES_OK(context, + context->allocate_output(0, TensorShape({}), + &output_sampled_index_tensor)); + auto output = output_sampled_index_tensor->scalar(); + // Return the last state of the Markov chain as the new center. + output() = selected_index; + } +}; + +REGISTER_KERNEL_BUILDER(Name("KMC2ChainInitialization").Device(DEVICE_CPU), + KMC2ChainInitializationOp); + // Operator for computing the nearest neighbors for a set of points. class NearestNeighborsOp : public OpKernel { public: diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc b/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc index c4a96b048db878169acc69b4d8caed5d4e04c18f..8172a7cebb81de70c530dbdd9ce0ca3eda4bc2ce 100644 --- a/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc +++ b/tensorflow/contrib/factorization/kernels/clustering_ops_test.cc @@ -116,6 +116,62 @@ RUN_BM_KmeansPlusPlusInitialization(k3RetriesPerSample); #undef RUN_BM_KmeansPlusPlusInitialization #undef BENCHMARK_KMEANS_PLUS_PLUS +Graph* SetUpKMC2Initialization(int num_points) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor distances(DT_FLOAT, TensorShape({num_points})); + Tensor seed(DT_INT64, TensorShape({})); + distances.flat().setRandom(); + seed.flat().setConstant(12345); + + TF_CHECK_OK( + NodeBuilder("KMC2ChainInitializationOp", "KMC2ChainInitialization") + .Input(test::graph::Constant(g, distances)) + .Input(test::graph::Constant(g, seed)) + .Finalize(g, nullptr /* node */)); + return g; +} + +template +void BM_KMC2Initialization(int iters) { + testing::StopTiming(); + testing::ItemsProcessed(static_cast(iters) * num_points * num_dims * + num_to_sample); + testing::UseRealTime(); + Graph* g = SetUpKMC2Initialization(num_points); + testing::StartTiming(); + test::Benchmark("cpu", g).Run(iters); +} +#define BENCHMARK_KMC2(p, c, d) \ + void BM_KMC2Initialization_##p##_##c##_##d(int iters) { \ + BM_KMC2Initialization(iters); \ + } \ + BENCHMARK(BM_KMC2Initialization_##p##_##c##_##d); + +#define RUN_BM_KMC2Initialization \ + BENCHMARK_KMC2(k10Points, k2Centers, k100Dim); \ + BENCHMARK_KMC2(k10Points, k5Centers, k100Dim); \ + BENCHMARK_KMC2(k10Points, k10Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k10Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k20Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k50Centers, k100Dim); \ + BENCHMARK_KMC2(k100Points, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k200Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k500Centers, k100Dim); \ + BENCHMARK_KMC2(k1kPoints, k1kCenters, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k200Centers, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k500Centers, k100Dim); \ + BENCHMARK_KMC2(k10kPoints, k1kCenters, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k100Centers, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k200Centers, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k500Centers, k100Dim); \ + BENCHMARK_KMC2(k1MPoints, k1kCenters, k100Dim) + +RUN_BM_KMC2Initialization; +#undef RUN_BM_KMC2Initialization +#undef BENCHMARK_KMC2 + Graph* SetUpNearestNeighbors(int num_dims, int num_points, int num_centers, int k) { Graph* g = new Graph(OpRegistry::Global()); diff --git a/tensorflow/contrib/factorization/ops/clustering_ops.cc b/tensorflow/contrib/factorization/ops/clustering_ops.cc index f2dfcf7ed0fb05264b10dee9980a246a5f2e49fa..2686702c1d5768f661dac610c96089eb02e360d7 100644 --- a/tensorflow/contrib/factorization/ops/clustering_ops.cc +++ b/tensorflow/contrib/factorization/ops/clustering_ops.cc @@ -44,6 +44,25 @@ num_retries_per_sample: Scalar. For each row that is sampled, this parameter samples: Matrix of shape (num_to_sample, d). The sampled rows. )"); +REGISTER_OP("KMC2ChainInitialization") + .Input("distances: float32") + .Input("seed: int64") + .Output("index: int64") + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"( +Returns the index of a data point that should be added to the seed set. + +Entries in distances are assumed to be squared distances of candidate points to +the already sampled centers in the seed set. The op constructs one Markov chain +of the k-MC^2 algorithm and returns the index of one candidate point to be added +as an additional cluster center. + +distances: Vector with squared distances to the closest previously sampled + cluster center for each candidate point. +seed: Scalar. Seed for initializing the random number generator. +index: Scalar with the index of the sampled point. +)"); + REGISTER_OP("NearestNeighbors") .Input("points: float32") .Input("centers: float32") diff --git a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py index 450f64063a2a357e422cd14761864d511c0e6cce..1322f7ce5f83d82c76040a30699137cd2bf491b5 100644 --- a/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py +++ b/tensorflow/contrib/factorization/python/kernel_tests/clustering_ops_test.py @@ -55,6 +55,63 @@ class KmeansPlusPlusInitializationTest(test.TestCase): self.runTestWithSeed(seed) +class KMC2InitializationTest(test.TestCase): + + def runTestWithSeed(self, seed): + with self.test_session(): + distances = np.zeros(1000).astype(np.float32) + distances[6] = 10e7 + distances[4] = 10e3 + + sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed) + self.assertEquals(sampled_point.eval(), 6) + distances[6] = 0.0 + sampled_point = clustering_ops.kmc2_chain_initialization(distances, seed) + self.assertEquals(sampled_point.eval(), 4) + + def testBasic(self): + for seed in range(100): + self.runTestWithSeed(seed) + + +class KMC2InitializationLargeTest(test.TestCase): + + def setUp(self): + self._distances = np.zeros(1001) + self._distances[500] = 100.0 + self._distances[1000] = 50.0 + + def testBasic(self): + with self.test_session(): + counts = {} + seed = 0 + for i in range(50): + sample = clustering_ops.kmc2_chain_initialization( + self._distances, seed + i).eval() + counts[sample] = counts.get(sample, 0) + 1 + self.assertEquals(len(counts), 2) + self.assertTrue(500 in counts) + self.assertTrue(1000 in counts) + self.assertGreaterEqual(counts[500], 5) + self.assertGreaterEqual(counts[1000], 5) + + +class KMC2InitializationCornercaseTest(test.TestCase): + + def setUp(self): + self._distances = np.zeros(10) + + def runTestWithSeed(self, seed): + with self.test_session(): + sampled_point = clustering_ops.kmc2_chain_initialization( + self._distances, seed) + self.assertEquals(sampled_point.eval(), 0) + + def testBasic(self): + for seed in range(100): + self.runTestWithSeed(seed) + + # A simple test that can be verified by hand. class NearestCentersTest(test.TestCase): diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py index d7320aeb3def08d23a256dcfee242bb4ecd9b6bd..96cc80ce241347ebca5b68140f1b1c8b9898ae72 100644 --- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py +++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py @@ -50,6 +50,7 @@ COSINE_DISTANCE = 'cosine' RANDOM_INIT = 'random' KMEANS_PLUS_PLUS_INIT = 'kmeans_plus_plus' +KMC2_INIT = 'kmc2' # The name of the variable holding the cluster centers. Used by the Estimator. CLUSTERS_VAR_NAME = 'clusters' @@ -66,7 +67,8 @@ class KMeans(object): use_mini_batch=False, mini_batch_steps_per_iteration=1, random_seed=0, - kmeans_plus_plus_num_retries=2): + kmeans_plus_plus_num_retries=2, + kmc2_chain_length=200): """Creates an object for generating KMeans clustering graph. This class implements the following variants of K-means algorithm: @@ -95,7 +97,8 @@ class KMeans(object): exactly like a full-batch version. Args: - inputs: An input tensor or list of input tensors + inputs: An input tensor or list of input tensors. It is assumed that the + data points have been previously randomly permuted. num_clusters: An integer tensor specifying the number of clusters. This argument is ignored if initial_clusters is a tensor or numpy array. initial_clusters: Specifies the clusters used during initialization. One @@ -104,6 +107,7 @@ class KMeans(object): - a function f(inputs, k) that returns up to k centers from `inputs`. - "random": Choose centers randomly from `inputs`. - "kmeans_plus_plus": Use kmeans++ to choose centers from `inputs`. + - "kmc2": Use the fast k-MC2 algorithm to choose centers from `inputs`. In the last three cases, one batch of `inputs` may not yield `num_clusters` centers, in which case initialization will require multiple batches until enough centers are chosen. In the case of @@ -121,13 +125,17 @@ class KMeans(object): additional points to draw from the current distribution before selecting the best. If a negative value is specified, a heuristic is used to sample O(log(num_to_sample)) additional points. + kmc2_chain_length: Determines how many candidate points are used by the + k-MC2 algorithm to produce one new cluster centers. If a (mini-)batch + contains less points, one new cluster center is generated from the + (mini-)batch. Raises: ValueError: An invalid argument was passed to initial_clusters or distance_metric. """ if isinstance(initial_clusters, str) and initial_clusters not in [ - RANDOM_INIT, KMEANS_PLUS_PLUS_INIT + RANDOM_INIT, KMEANS_PLUS_PLUS_INIT, KMC2_INIT ]: raise ValueError( "Unsupported initialization algorithm '%s'" % initial_clusters) @@ -141,6 +149,7 @@ class KMeans(object): self._mini_batch_steps_per_iteration = int(mini_batch_steps_per_iteration) self._random_seed = random_seed self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries + self._kmc2_chain_length = kmc2_chain_length @classmethod def _distance_graph(cls, inputs, clusters, distance_metric): @@ -302,9 +311,10 @@ class KMeans(object): else: cluster_centers_updated = cluster_centers update_in_steps = None - cluster_counts = (variable_scope.variable( - array_ops.ones([num_clusters], dtype=dtypes.int64)) - if self._use_mini_batch else None) + cluster_counts = ( + variable_scope.variable( + array_ops.ones([num_clusters], dtype=dtypes.int64)) + if self._use_mini_batch else None) return (cluster_centers, cluster_centers_initialized, cluster_counts, cluster_centers_updated, update_in_steps) @@ -359,7 +369,7 @@ class KMeans(object): init_op = _InitializeClustersOpFactory( self._inputs, num_clusters, initial_clusters, self._distance_metric, self._random_seed, self._kmeans_plus_plus_num_retries, - cluster_centers_var, cluster_centers_updated, + self._kmc2_chain_length, cluster_centers_var, cluster_centers_updated, cluster_centers_initialized).op() cluster_centers = cluster_centers_var @@ -520,8 +530,9 @@ class KMeans(object): array_ops.reshape(array_ops.shape(inp)[0], [-1])), [-1, 1]), cluster_idx, num_clusters)) with ops.colocate_with(cluster_centers, ignore_existing=True): - new_clusters_centers = math_ops.add_n(cluster_sums) / (math_ops.cast( - math_ops.add_n(cluster_counts), cluster_sums[0].dtype) + epsilon) + new_clusters_centers = math_ops.add_n(cluster_sums) / ( + math_ops.cast(math_ops.add_n(cluster_counts), cluster_sums[0].dtype) + + epsilon) if self._clusters_l2_normalized(): new_clusters_centers = nn_impl.l2_normalize(new_clusters_centers, dim=1) return state_ops.assign(cluster_centers, new_clusters_centers) @@ -548,9 +559,12 @@ class _InitializeClustersOpFactory(object): cluster_centers_initialized := true """ + # TODO(ccolby): Refactor this class so that kmc2 isn't so much a special case. + def __init__(self, inputs, num_clusters, initial_clusters, distance_metric, - random_seed, kmeans_plus_plus_num_retries, cluster_centers, - cluster_centers_updated, cluster_centers_initialized): + random_seed, kmeans_plus_plus_num_retries, kmc2_chain_length, + cluster_centers, cluster_centers_updated, + cluster_centers_initialized): """Creates an op factory. Args: @@ -560,6 +574,7 @@ class _InitializeClustersOpFactory(object): distance_metric: See KMeans constructor. random_seed: See KMeans constructor. kmeans_plus_plus_num_retries: See KMeans constructor. + kmc2_chain_length: See KMeans constructor. cluster_centers: The TF variable holding the initial centers. It may already contain some centers when the op is executed. cluster_centers_updated: A second TF variable to hold a copy of the @@ -575,6 +590,7 @@ class _InitializeClustersOpFactory(object): self._distance_metric = distance_metric self._random_seed = random_seed self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries + self._kmc2_chain_length = kmc2_chain_length self._cluster_centers = cluster_centers self._cluster_centers_updated = cluster_centers_updated self._cluster_centers_initialized = cluster_centers_initialized @@ -604,6 +620,90 @@ class _InitializeClustersOpFactory(object): math_ops.to_int64(self._num_remaining), self._random_seed, self._kmeans_plus_plus_num_retries) + def _kmc2_multiple_centers(self): + """Adds new initial cluster centers using the k-MC2 algorithm. + + In each call to the op, the provided batch is split into subsets based on + the specified `kmc2_chain_length`. On each subset, a single Markov chain of + the k-MC2 algorithm is used to add *one* new center cluster center. If there + are less than `kmc2_chain_length` points in the subset, a single center is + added using one Markov chain on the full input. It is assumed that the + provided batch has previously been randomly permuted. Otherwise, k-MC2 may + return suboptimal centers. + + Returns: + An op that adds new cluster centers. + """ + # The op only operates on the first shard of data. + first_shard = self._inputs[0] + # Number of points in the input that can be used. + batch_size = array_ops.shape(first_shard)[0] + # Maximum number of subsets such that the size of each subset is at least + # `kmc2_chain_length`. Final subsets may be larger. + max_to_sample = math_ops.cast( + batch_size / self._kmc2_chain_length, dtype=dtypes.int32) + # We sample at least one new center and at most all remaining centers. + num_to_sample = math_ops.maximum( + math_ops.minimum(self._num_remaining, max_to_sample), 1) + + def _cond(i, _): + """Stopping condition for the while loop.""" + return math_ops.less(i, num_to_sample) + + def _body(i, _): + """Body that adds a single new center based on a subset.""" + + def _sample_random(): + """Returns a random point as a cluster center.""" + # By assumption the batch is reshuffled and _sample_random is always + # called for i=0. Hence, we simply return the first point. + new_center = array_ops.reshape(first_shard[0], [1, -1]) + if self._distance_metric == COSINE_DISTANCE: + new_center = nn_impl.l2_normalize(new_center, dim=1) + return new_center + + def _sample_kmc2_chain(): + """Returns previous centers as well as a new center sampled using k-MC2. + """ + # Extract the subset from the underlying batch. + start = i * self._kmc2_chain_length + end = start + self._kmc2_chain_length + subset = first_shard[start:end] + # Compute the distances from points in the subset to previous centers. + _, distances = gen_clustering_ops.nearest_neighbors( + subset, self._cluster_centers, 1) + # Sample index of new center using k-MC2 Markov chain. + new_center_index = gen_clustering_ops.kmc2_chain_initialization( + array_ops.squeeze(distances), self._random_seed) + # Extract actual new center. + newly_sampled_center = array_ops.reshape(subset[new_center_index], + [1, -1]) + # Return concatenation with previously sampled centers. + if self._distance_metric == COSINE_DISTANCE: + newly_sampled_center = nn_impl.l2_normalize( + newly_sampled_center, dim=1) + return array_ops.concat([self._cluster_centers, newly_sampled_center], + 0) + + # Obtain a random point if there are no previously sampled centers. + # Otherwise, construct a k-MC2 Markov chain. + new_centers = control_flow_ops.cond( + math_ops.equal(self._num_selected, 0), _sample_random, + _sample_kmc2_chain) + # Assign new cluster centers to underlying variable. + assigned_centers = state_ops.assign( + self._cluster_centers, new_centers, validate_shape=False) + if self._cluster_centers_updated is not self._cluster_centers: + assigned_centers = state_ops.assign( + self._cluster_centers_updated, + assigned_centers, + validate_shape=False) + return i + 1, self._num_clusters - array_ops.shape(assigned_centers)[0] + + # Add num_to_sample new data points. + _, num_remaining = control_flow_ops.while_loop(_cond, _body, [0, 0]) + return num_remaining + def _greedy_batch_sampler(self, sampler): # If the input dataset size is smaller than the number of centers # remaining, choose the entire input dataset as centers. This can happen @@ -657,7 +757,10 @@ class _InitializeClustersOpFactory(object): with ops.control_dependencies([ check_ops.assert_positive(self._num_remaining), ]): - num_now_remaining = self._add_new_centers() + if self._initial_clusters == KMC2_INIT: + num_now_remaining = self._kmc2_multiple_centers() + else: + num_now_remaining = self._add_new_centers() return control_flow_ops.cond( math_ops.equal(num_now_remaining, 0), lambda: state_ops.assign(self._cluster_centers_initialized, True), diff --git a/tensorflow/contrib/ffmpeg/BUILD b/tensorflow/contrib/ffmpeg/BUILD index e205d92fbe2f45cafde76f79643eb85b6876d48b..7a5a4cb8c9499b950a3ad89be710e48474d5791e 100644 --- a/tensorflow/contrib/ffmpeg/BUILD +++ b/tensorflow/contrib/ffmpeg/BUILD @@ -89,6 +89,7 @@ tf_py_test( "@six_archive//:six", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", "//tensorflow/python:platform", ], data = [ @@ -105,6 +106,7 @@ tf_py_test( "@six_archive//:six", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", "//tensorflow/python:platform", ], data = [ diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD index 05fc658d80f26b00f775211cf89f55ce18a4502d..949ae9ad9e4b045ee1b5cc82d49c0e7468c2005d 100644 --- a/tensorflow/contrib/ffmpeg/default/BUILD +++ b/tensorflow/contrib/ffmpeg/default/BUILD @@ -23,6 +23,18 @@ cc_library( ], ) +tf_cc_test( + name = "ffmpeg_lib_utility_test", + srcs = ["ffmpeg_lib_utility_test.cc"], + deps = [ + ":ffmpeg_lib", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + tf_cc_test( name = "ffmpeg_lib_installed_test", srcs = ["ffmpeg_lib_test.cc"], diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc index b417a70b6e63310a5a1d9a82522cd5e678e7b6b0..545a4386d043af604a747b8b5a8103101812b177 100644 --- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc +++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc @@ -198,6 +198,14 @@ string BuildWavFile(int32 samples_per_second, int32 channel_count, return data; } +// Returns a unique number every time it is called. +int64 UniqueId() { + static mutex mu(LINKER_INITIALIZED); + static int64 id = 0; + mutex_lock l(mu); + return ++id; +} + } // namespace string GetTempFilename(const string& extension) { @@ -208,8 +216,12 @@ string GetTempFilename(const string& extension) { } struct stat statbuf; if (!stat(dir, &statbuf) && S_ISDIR(statbuf.st_mode)) { - string tmp_filepath = - io::JoinPath(dir, StrCat("tmp_file_XXXXXX", ".", extension)); + // UniqueId is added here because mkstemps is not as thread safe as it + // looks. https://github.com/tensorflow/tensorflow/issues/5804 shows + // the problem. + string tmp_filepath = io::JoinPath( + dir, + StrCat("tmp_file_tensorflow_", UniqueId(), "_XXXXXX.", extension)); int fd = mkstemps(&tmp_filepath[0], extension.length() + 1); if (fd < 0) { LOG(FATAL) << "Failed to create temp file."; diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..7176f3b550679555d5ab3b70f2b360a90eaee253 --- /dev/null +++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib_utility_test.cc @@ -0,0 +1,80 @@ +// Copyright 2017 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +#include "tensorflow/contrib/ffmpeg/ffmpeg_lib.h" + +#include +#include +#include +#include + +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace ffmpeg { +namespace { + +TEST(FfmpegLibTest, TestTempDirectoryThreading) { + // Testing a fix for a bug that allowed different threads to create + // conflicting temp files. + // See github.com/tensorflow/tensorflow/issues/5804 for details. + const int32 kNumThreads = 10; + const int32 kNumWorkItems = 10000; + static constexpr size_t kStringsPerItem = 100; + Env* environment = Env::Default(); + thread::ThreadPool pool(environment, "test", kNumThreads); + + mutex mu; + std::vector temp_filenames; + temp_filenames.reserve(kNumWorkItems * kStringsPerItem); + + // Queue a large number of work items for the threads to process. Each work + // item creates a temp file and then deletes it. + for (int i = 0; i < kNumWorkItems; ++i) { + pool.Schedule([&mu, &temp_filenames, environment]() { + std::array buffer; + for (int32 j = 0; j < kStringsPerItem; ++j) { + buffer[j] = GetTempFilename("mp3"); + TF_QCHECK_OK(environment->DeleteFile(buffer[j])); + } + mutex_lock l(mu); + for (const auto& fn : buffer) { + temp_filenames.push_back(fn); + } + }); + } + + // Wait until all work items are complete. + while (true) { + mutex_lock l(mu); + if (temp_filenames.size() == kNumWorkItems * kStringsPerItem) { + break; + } + } + + // Check that no duplicates are created. + std::set unique_filenames; + mutex_lock l(mu); + for (const auto& fn : temp_filenames) { + ASSERT_TRUE(unique_filenames.insert(fn).second); + } +} + +} // namespace +} // namespace ffmpeg +} // namespace tensorflow diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD index dd882acb8ee35a91f2e67511b1465b3a561d72a6..891425fd8cae6fbbf60d30cbd9137c049073456c 100644 --- a/tensorflow/contrib/framework/BUILD +++ b/tensorflow/contrib/framework/BUILD @@ -47,6 +47,7 @@ tf_custom_op_py_library( "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", "//tensorflow/python:audio_ops_gen", + "//tensorflow/python:check_ops", "//tensorflow/python:checkpoint_ops_gen", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework", @@ -56,13 +57,17 @@ tf_custom_op_py_library( "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", "//tensorflow/python:state_ops_gen", "//tensorflow/python:tensor_array_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", "//third_party/py/numpy", "@six_archive//:six", ], @@ -158,6 +163,11 @@ py_test( ":framework_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:gradients", + "//tensorflow/python:platform_test", + "//tensorflow/python:variables", + "//third_party/py/numpy", ], ) @@ -170,7 +180,14 @@ py_test( ":framework_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:gradients", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python/eager:backprop", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:tape", + "//third_party/py/numpy", ], ) @@ -239,7 +256,6 @@ py_test( deps = [ ":framework_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", @@ -247,6 +263,7 @@ py_test( "//tensorflow/python:nn_ops", "//tensorflow/python:partitioned_variables", "//tensorflow/python:platform", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", @@ -279,7 +296,6 @@ py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", - "//tensorflow/python:errors", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:partitioned_variables", diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py index 2081a11f47d71106f8e57227f46639717a791855..8421ba7c0423c6ed274f92ba74930822d0171e05 100644 --- a/tensorflow/contrib/framework/__init__.py +++ b/tensorflow/contrib/framework/__init__.py @@ -37,6 +37,7 @@ See the @{$python/contrib.framework} guide. @@arg_scope @@add_arg_scope +@@current_arg_scope @@has_arg_scope @@arg_scoped_arguments diff --git a/tensorflow/contrib/framework/python/framework/tensor_util.py b/tensorflow/contrib/framework/python/framework/tensor_util.py index 92a2a4ff2d1cb41c48312038d82be0b6136f8d41..4e6eea8884731f3e14a7ae817296c3782d943527 100644 --- a/tensorflow/contrib/framework/python/framework/tensor_util.py +++ b/tensorflow/contrib/framework/python/framework/tensor_util.py @@ -77,10 +77,10 @@ def reduce_sum_n(tensors, name=None): return tensors[0] return math_ops.add_n(tensors, name=name_scope) -@deprecated(None, - 'Please switch to tf.confusion_matrix.remove_squeezable_dimensions.' - 'Note that order of the inputs and outputs of labels and ' - 'predictions have also been switched.') +@deprecated( + None, "Please switch to remove_squeezable_dimensions from " + "tf.confusion_matrix. Note that the order of the inputs and outputs of " + "labels and predictions have also been switched.") def remove_squeezable_dimensions(predictions, labels, name=None): """Squeeze last dim if ranks of `predictions` and `labels` differ by 1. diff --git a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py index 8c618838bfbcd1b0572c3a57aa6b27c68ee34f0c..c2229bb8ad3d5b38321d16f150ed94175ab9bdbe 100644 --- a/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py +++ b/tensorflow/contrib/framework/python/ops/accumulate_n_v2_eager_test.py @@ -64,7 +64,8 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase): np.random.seed(42) num_inputs = 3 input_vars = [ - resource_variable_ops.ResourceVariable(10.0 * np.random.random()) + resource_variable_ops.ResourceVariable(10.0 * np.random.random(), + name="t%d" % i) for i in range(0, num_inputs) ] @@ -79,6 +80,6 @@ class AccumulateNV2EagerTest(test_util.TensorFlowTestCase): if __name__ == "__main__": - eager_context.enable_eager_execution() + ops.enable_eager_execution() test.main() diff --git a/tensorflow/contrib/framework/python/ops/arg_scope.py b/tensorflow/contrib/framework/python/ops/arg_scope.py index 9c194ec202ab6150278b26e844b9d3e97a7d6761..2bce00fde2459878a12027bb4d98bd3818bc92a2 100644 --- a/tensorflow/contrib/framework/python/ops/arg_scope.py +++ b/tensorflow/contrib/framework/python/ops/arg_scope.py @@ -67,6 +67,7 @@ from tensorflow.python.util import tf_decorator __all__ = ['arg_scope', 'add_arg_scope', + 'current_arg_scope', 'has_arg_scope', 'arg_scoped_arguments'] @@ -83,7 +84,7 @@ def _get_arg_stack(): return _ARGSTACK -def _current_arg_scope(): +def current_arg_scope(): stack = _get_arg_stack() return stack[-1] @@ -144,7 +145,7 @@ def arg_scope(list_ops_or_scope, **kwargs): raise TypeError('list_ops_or_scope must either be a list/tuple or reused' 'scope (i.e. dict)') try: - current_scope = _current_arg_scope().copy() + current_scope = current_arg_scope().copy() for op in list_ops_or_scope: key_op = _key_op(op) if not has_arg_scope(op): @@ -172,7 +173,7 @@ def add_arg_scope(func): A tuple with the decorated function func_with_args(). """ def func_with_args(*args, **kwargs): - current_scope = _current_arg_scope() + current_scope = current_arg_scope() current_args = kwargs key_func = _key_op(func) if key_func in current_scope: diff --git a/tensorflow/contrib/fused_conv/BUILD b/tensorflow/contrib/fused_conv/BUILD index 31917b40eb900dd6a0a6c1a83d00881dfe690c49..ce37672895b37275770d2f5410f662e9acf1de9d 100644 --- a/tensorflow/contrib/fused_conv/BUILD +++ b/tensorflow/contrib/fused_conv/BUILD @@ -38,7 +38,6 @@ tf_custom_op_py_library( ":fused_conv2d_bias_activation_op", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:control_flow_ops", @@ -49,6 +48,7 @@ tf_custom_op_py_library( "//tensorflow/python:nn_ops", "//tensorflow/python:platform", "//tensorflow/python:random_ops", + "//tensorflow/python:session", "//tensorflow/python:util", "//tensorflow/python:variables", "//third_party/py/numpy", @@ -69,7 +69,7 @@ tf_kernel_library( "//tensorflow/core:lib", "//tensorflow/core:lib_proto_parsing", "//tensorflow/core:stream_executor", - "//tensorflow/core/kernels:bounds_check_lib", + "//tensorflow/core/kernels:bounds_check", "//tensorflow/core/kernels:conv_2d_hdrs", "//tensorflow/core/kernels:conv_ops_gpu_hdrs", "//tensorflow/core/kernels:gpu_util_hdrs", diff --git a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc index e4c39739f7fc653b68e82c994fc69e3e168f65f9..88306094ab9947c9c78b03c0013f6afc88316803 100644 --- a/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc +++ b/tensorflow/contrib/fused_conv/kernels/fused_conv2d_bias_activation_op.cc @@ -445,11 +445,11 @@ void LaunchFusedConv2DBiasActivationOp:: .set_zero_padding_width(padding_cols / 2); Tensor maybe_transformed_filter; - const Tensor* filter; - if (is_int8x4) { - // We have already checked filter is OIHW_VECT_I in the constructor. - filter = &filter_param; - } else if (filter_format == FORMAT_HWIO) { + const Tensor* filter = &filter_param; + // For qint8, we have already checked filter is OIHW_VECT_I in the + // constructor, but we need to test for is_int8x4 so the if block doesn't + // generate code for qint8. + if (!is_int8x4 && filter_format == FORMAT_HWIO) { // Shuffle filter tensor from HWIO to OIHW: OP_REQUIRES_OK(ctx, ctx->allocate_temp( DataTypeToEnum::value, diff --git a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py index 3b8f7d6ed760647c4c61ce5ea60be1d7d17ddfa0..2a18f3eeecc7e0e69c54b219886a263136f01b2c 100644 --- a/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py +++ b/tensorflow/contrib/fused_conv/python/ops/fused_conv2d_bias_activation_op_test.py @@ -159,9 +159,12 @@ class FusedConv2DBiasActivationTest(test.TestCase): def _DtypesToTest(self, use_gpu): return [dtypes.float32] + def _FilterFormatsToTest(self, use_gpu): + return ["HWIO", "OIHW"] + def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, bias, strides, padding, activation_mode, data_format, - dtype): + filter_format, dtype): """Verifies the output values of the convolution function. Args: @@ -174,6 +177,7 @@ class FusedConv2DBiasActivationTest(test.TestCase): padding: Padding type. activation_mode: Activation mode. data_format: Format of the data tensors. + filter_format: Filter format to use for the fused convolution. dtype: Data type for inputs and outputs. Returns: Symbolic tensor value and reference value that can be used to @@ -192,6 +196,9 @@ class FusedConv2DBiasActivationTest(test.TestCase): with self.test_session(use_gpu=True): t1 = constant_op.constant(x1, shape=tensor_in_sizes, dtype=dtype) t2 = constant_op.constant(x2, shape=filter_in_sizes, dtype=dtype) + fused_t2 = t2 + if filter_format == "OIHW": + fused_t2 = HwioToOihw(t2) t3 = constant_op.constant(x3, shape=[bias_size], dtype=dtype) strides = [1] + strides + [1] if data_format == "NCHW": @@ -199,11 +206,12 @@ class FusedConv2DBiasActivationTest(test.TestCase): strides = test_util.NHWCToNCHW(strides) output = fused_conv2d_bias_activation_op.fused_conv2d_bias_activation( t1, - t2, + fused_t2, t3, strides=strides, padding=padding, data_format=data_format, + filter_format=filter_format, activation_mode=activation_mode) ref_conv_output = nn_ops.conv2d( t1, t2, strides=strides, padding=padding, data_format=data_format) @@ -268,9 +276,10 @@ class FusedConv2DBiasActivationTest(test.TestCase): ref_tensors = [] for (data_format, use_gpu) in GetTestConfigs(): for dtype in self._DtypesToTest(use_gpu): - result, expected = self._SetupValuesForDevice( - tensor_in_sizes, filter_in_sizes, bias, strides, padding, "Relu", - data_format, dtype) + for filter_format in self._FilterFormatsToTest(use_gpu): + result, expected = self._SetupValuesForDevice( + tensor_in_sizes, filter_in_sizes, bias, strides, padding, "Relu", + data_format, filter_format, dtype) tensors.append(result) ref_tensors.append(expected) with self.test_session() as sess: @@ -607,6 +616,10 @@ def NchwToNchwVectC(in_tensor): return array_ops.transpose(t, [0, 1, 3, 4, 2]) +def HwioToOihw(in_tensor): + return array_ops.transpose(in_tensor, [3, 2, 0, 1]) + + def SimulateFusedConv2dBiasActivationInt8(conv_input_scale, conv_input, kernel, padding, strides, side_input_scale, side_input, biases): diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD index 27a5d6ec31f0df5f0f3a435185f50a6c88122b19..1418c87023af0dbff890f46e10f0140d5b89e4b7 100644 --- a/tensorflow/contrib/gan/BUILD +++ b/tensorflow/contrib/gan/BUILD @@ -202,6 +202,7 @@ py_library( "//tensorflow/python:embedding_ops", "//tensorflow/python:math_ops", "//tensorflow/python:tensor_util", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", ], ) @@ -234,6 +235,7 @@ py_library( "//tensorflow/python:nn", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", ], ) @@ -267,7 +269,10 @@ py_library( "python/features/python/clip_weights_impl.py", ], srcs_version = "PY2AND3", - deps = ["//tensorflow/contrib/opt:opt_py"], + deps = [ + "//tensorflow/contrib/opt:opt_py", + "//tensorflow/python:util", + ], ) py_test( diff --git a/tensorflow/contrib/gan/README.md b/tensorflow/contrib/gan/README.md index 10458a2458384c8f589183003256db24d69742d7..3ab84780705b35567169bd76fd3485ad355ba9d8 100644 --- a/tensorflow/contrib/gan/README.md +++ b/tensorflow/contrib/gan/README.md @@ -47,13 +47,14 @@ such as the Wasserstein loss, gradient penalty, mutual information penalty, etc * [evaluation](https://www.tensorflow.org/code/tensorflow/contrib/gan/python/eval/python/): Use `Inception Score` or `Frechet Distance` with a pretrained Inception -network to evaluate your unconditional generative model. You can also also use +network to evaluate your unconditional generative model. You can also use your own pretrained classifier for more specific performance numbers, or use other methods for evaluating conditional generative models. -* [examples](https://github.com/tensorflow/models/tree/master/gan/): +* examples (coming soon): See examples of how to use TFGAN to make GAN training easier, or use the more complicated examples to jumpstart your -own project. +own project. These include unconditional and conditional GANs, InfoGANs, +adversarial losses on existing networks, and image-to-image translation. ## Training a GAN model diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py index 6074694f8b87f65a2b2f8a3c4d7ac6b93482afd3..d4c080cab3d82f6a69a293e84e1c08322bbb6f86 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py @@ -16,6 +16,11 @@ These methods come from https://arxiv.org/abs/1606.03498 and https://arxiv.org/abs/1706.08500. + +NOTE: This implementation uses the same weights as in +https://github.com/openai/improved-gan/blob/master/inception_score/model.py, +but is more numerically stable and is an unbiased estimator of the true +Inception score even when splitting the inputs into batches. """ from __future__ import absolute_import @@ -54,17 +59,16 @@ __all__ = [ 'classifier_score', 'frechet_inception_distance', 'frechet_classifier_distance', + 'INCEPTION_DEFAULT_IMAGE_SIZE', ] -INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v3_2017_09_13.tar.gz' -INCEPTION_FROZEN_GRAPH = 'frozen_inception_v3.pb' -INCEPTION_V3_INPUT = 'input' -INCEPTION_V3_OUTPUT = 'InceptionV3/Logits/SpatialSqueeze:0' -INCEPTION_V3_FINAL_POOL = 'InceptionV3/Logits/AvgPool_1a_8x8/AvgPool:0' -_INCEPTION_V3_NUM_CLASSES = 1001 -_INCEPTION_V3_FINAL_POOL_SIZE = 2048 -INCEPTION_V3_DEFAULT_IMG_SIZE = 299 +INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v1_2015_12_05.tar.gz' +INCEPTION_FROZEN_GRAPH = 'inceptionv1_for_inception_score.pb' +INCEPTION_INPUT = 'Mul:0' +INCEPTION_OUTPUT = 'logits:0' +INCEPTION_FINAL_POOL = 'pool_3:0' +INCEPTION_DEFAULT_IMAGE_SIZE = 299 def _validate_images(images, image_size): @@ -102,46 +106,37 @@ def _symmetric_matrix_square_root(mat, eps=1e-10): math_ops.matmul(u, array_ops.diag(si)), v, transpose_b=True) -# Convenience preprocessing function, with fixed defaults. -# NOTE: Floating-point inputs are expected to be in [0, 1]. -# Copied from /tensorflow_models/slim/preprocessing/inception_preprocessing.py. def preprocess_image( - image, height=INCEPTION_V3_DEFAULT_IMG_SIZE, - width=INCEPTION_V3_DEFAULT_IMG_SIZE, central_fraction=0.875, scope=None): - """Prepare one image for evaluation. + images, height=INCEPTION_DEFAULT_IMAGE_SIZE, + width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None): + """Prepare a batch of images for evaluation. - If height and width are specified it would output an image with that size by - applying resize_bilinear. + This is the preprocessing portion of the graph from + http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz. - If central_fraction is specified it would crop the central fraction of the - input image. + Note that it expects Tensors in [0, 255]. This function maps pixel values to + [-1, 1] and resizes to match the InceptionV1 network. Args: - image: 3-D Tensor of image. If dtype is tf.float32 then the range should be - [0, 1], otherwise it would converted to tf.float32 assuming that the range - is [0, MAX], where MAX is largest positive representable number for - int(8/16/32) data type (see `tf.image.convert_image_dtype` for details). - height: integer - width: integer - central_fraction: Optional Float, fraction of the image to crop. + images: 3-D or 4-D Tensor of images. Values are in [0, 255]. + height: Integer. Height of resized output image. + width: Integer. Width of resized output image. scope: Optional scope for name_scope. + Returns: - 3-D float Tensor of prepared image. + 3-D or 4-D float Tensor of prepared image(s). Values are in [-1, 1]. """ - with ops.name_scope(scope, 'eval_image', [image, height, width]): - if image.dtype != dtypes.float32: - image = image_ops.convert_image_dtype(image, dtype=dtypes.float32) - # Crop the central region of the image with an area containing 87.5% of - # the original image. - image = image_ops.central_crop(image, central_fraction=central_fraction) - - # Resize the image to the specified height and width. - image = array_ops.expand_dims(image, 0) - image = image_ops.resize_bilinear(image, [height, width], - align_corners=False) - image = array_ops.squeeze(image, [0]) - image = (image - 0.5) * 2.0 - return image + is_single = images.shape.ndims == 3 + with ops.name_scope(scope, 'preprocess', [images, height, width]): + if not images.dtype.is_floating: + images = math_ops.to_float(images) + images = (images - 128.0) / 128.0 + if is_single: + images = array_ops.expand_dims(images, axis=0) + resized = image_ops.resize_bilinear(images, [height, width]) + if is_single: + resized = array_ops.squeeze(resized, axis=0) + return resized def _kl_divergence(p, p_logits, q): @@ -211,9 +206,9 @@ def _default_graph_def_fn(): def run_inception(images, graph_def=None, default_graph_def_fn=_default_graph_def_fn, - image_size=INCEPTION_V3_DEFAULT_IMG_SIZE, - input_tensor=INCEPTION_V3_INPUT, - output_tensor=INCEPTION_V3_OUTPUT): + image_size=INCEPTION_DEFAULT_IMAGE_SIZE, + input_tensor=INCEPTION_INPUT, + output_tensor=INCEPTION_OUTPUT): """Run images through a pretrained Inception classifier. Args: @@ -338,7 +333,7 @@ def classifier_score(images, classifier_fn, num_batches=1): inception_score = functools.partial( classifier_score, classifier_fn=functools.partial( - run_inception, output_tensor=INCEPTION_V3_OUTPUT)) + run_inception, output_tensor=INCEPTION_OUTPUT)) def trace_sqrt_product(sigma, sigma_v): @@ -479,4 +474,4 @@ def frechet_classifier_distance(real_images, frechet_inception_distance = functools.partial( frechet_classifier_distance, classifier_fn=functools.partial( - run_inception, output_tensor=INCEPTION_V3_FINAL_POOL)) + run_inception, output_tensor=INCEPTION_FINAL_POOL)) diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py index 30285964a53c388d4f9aaf65b6cabed362b3b012..81fa2fc0f126647d2f01a1f4fc695d714eba2c75 100644 --- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py +++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py @@ -68,7 +68,7 @@ def _expected_trace_sqrt_product(sigma, sigma_v): # A dummy GraphDef string with the minimum number of Ops. graphdef_string = """ node { - name: "input" + name: "Mul" op: "Placeholder" attr { key: "dtype" @@ -97,7 +97,7 @@ node { } } node { - name: "InceptionV3/Logits/SpatialSqueeze" + name: "logits" op: "Placeholder" attr { key: "dtype" @@ -120,7 +120,7 @@ node { } } node { - name: "InceptionV3/Logits/AvgPool_1a_8x8/AvgPool" + name: "pool_3" op: "Placeholder" attr { key: "dtype" @@ -182,7 +182,7 @@ class ClassifierMetricsTest(test.TestCase): img = array_ops.ones([batch_size, 299, 299, 3]) pool = _run_with_mock( classifier_metrics.run_inception, img, - output_tensor=classifier_metrics.INCEPTION_V3_FINAL_POOL) + output_tensor=classifier_metrics.INCEPTION_FINAL_POOL) self.assertTrue(isinstance(pool, ops.Tensor)) pool.shape.assert_is_compatible_with([batch_size, 2048]) @@ -306,7 +306,7 @@ class ClassifierMetricsTest(test.TestCase): """Test `preprocess_image` graph construction.""" incorrectly_sized_image = array_ops.zeros([520, 240, 3]) correct_image = classifier_metrics.preprocess_image( - image=incorrectly_sized_image) + images=incorrectly_sized_image) _run_with_mock(classifier_metrics.run_inception, array_ops.expand_dims(correct_image, 0)) diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD index bebcf079ba444946bf0377106cbafcbaa7e94e74..a417dba87543d82526ab856e5b915ee47f496d46 100644 --- a/tensorflow/contrib/gdr/BUILD +++ b/tensorflow/contrib/gdr/BUILD @@ -85,7 +85,6 @@ tf_cuda_library( "//tensorflow/core/distributed_runtime:rendezvous_mgr_interface", "//tensorflow/core/distributed_runtime:worker", "//tensorflow/core/distributed_runtime:worker_cache", - "//tensorflow/core/distributed_runtime:worker_env", "//tensorflow/core/distributed_runtime:worker_session", "//tensorflow/core/distributed_runtime/rpc:grpc_call", "//tensorflow/core/distributed_runtime/rpc:grpc_tensor_coding", @@ -119,7 +118,6 @@ cc_library( ":gdr_memory_manager", ":gdr_rendezvous_mgr", ":gdr_worker", - "//tensorflow/core:lib_internal", "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib", ], alwayslink = 1, diff --git a/tensorflow/contrib/graph_editor/BUILD b/tensorflow/contrib/graph_editor/BUILD index b4c53d3da655e2f52b5990ac0de3bc7ccc823bcc..967ad2fc090906e93f22c777816eede37f9a1b04 100644 --- a/tensorflow/contrib/graph_editor/BUILD +++ b/tensorflow/contrib/graph_editor/BUILD @@ -144,12 +144,12 @@ py_test( ":graph_editor_py", ":match", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:gradients", "//tensorflow/python:math_ops", + "//tensorflow/python:session", "//tensorflow/python:variables", "//third_party/py/numpy", ], diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py index ab5776b9dd66bb082e9ca3922e8902bfebe6b0b8..ca00394388f67e2ed9508684a47b23c3ee9e79e8 100644 --- a/tensorflow/contrib/graph_editor/tests/transform_test.py +++ b/tensorflow/contrib/graph_editor/tests/transform_test.py @@ -191,14 +191,14 @@ class TransformTest(test.TestCase): # Extract the operations. replacement_ts = {w.value(): g} original_mul1_grad = (ops.get_default_graph(). - get_operation_by_name("grad/mul1_grad/mul_1")) + get_operation_by_name("grad/mul1_grad/Mul_1")) # Should not raise exception. res = ge.graph_replace(g, replacement_ts, dst_scope="res") # Extract the operations after graph_replace. result_mul1_grad = (ops.get_default_graph(). - get_operation_by_name("res/grad/mul1_grad/mul_1")) + get_operation_by_name("res/grad/mul1_grad/Mul_1")) # Make sure _original_ops are as expected. self.assertEquals(original_mul1_grad._original_op.name, u"mul1") diff --git a/tensorflow/contrib/grid_rnn/BUILD b/tensorflow/contrib/grid_rnn/BUILD index 7fbb9f024c589895aa2dff7b6f5d8ba8c399af48..d601a1ec6f7a219bcd461d819ab2dfc64135a3ae 100644 --- a/tensorflow/contrib/grid_rnn/BUILD +++ b/tensorflow/contrib/grid_rnn/BUILD @@ -31,14 +31,12 @@ cuda_py_tests( additional_deps = [ ":grid_rnn_py", "//third_party/py/numpy", - "//tensorflow/contrib/rnn:rnn_py", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:framework_test_lib", "//tensorflow/python:init_ops", "//tensorflow/python:nn_ops", - "//tensorflow/python:platform_test", + "//tensorflow/python:rnn", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/hooks/BUILD b/tensorflow/contrib/hooks/BUILD index 1576c9ec9b3e058091fd7db865c0368b53d9d3cb..1b528d7afc1112f5dc0667ae299ade02bc8fd04b 100644 --- a/tensorflow/contrib/hooks/BUILD +++ b/tensorflow/contrib/hooks/BUILD @@ -20,6 +20,7 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/python:training", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD index 8c92e33bdf01a5aec33892fe140da5f762f05679..324035100df366b80f57af9052c4bd935655b248 100644 --- a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD +++ b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD @@ -52,13 +52,9 @@ tf_cc_binary( "//tensorflow/core:android_tensorflow_test_lib", ], "//conditions:default": [ - "//tensorflow/core:core_cpu", "//tensorflow/core:lib", - "//tensorflow/core:framework", "//tensorflow/core:framework_internal", - "//tensorflow/core:protos_all_cc", "//tensorflow/core:tensorflow", - "//tensorflow/core:test", ], }), ) diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD index d0600d4668b2943e7fc880a079750d3a59406d68..157e97d237021d95c935a6be66aa57842b97125c 100755 --- a/tensorflow/contrib/image/BUILD +++ b/tensorflow/contrib/image/BUILD @@ -75,11 +75,13 @@ tf_custom_op_py_library( ":image_ops", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", + "//tensorflow/python:common_shapes", "//tensorflow/python:constant_op", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:util", ], ) @@ -143,12 +145,13 @@ py_library( srcs_version = "PY2AND3", deps = [ ":distort_image_ops", + ":single_image_random_dot_stereograms_py", "//tensorflow/contrib/util:util_py", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:image_ops", "//tensorflow/python:platform", "//tensorflow/python:random_ops", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/input_pipeline/BUILD b/tensorflow/contrib/input_pipeline/BUILD index bb7857eb998beb89517985a401d5b7afe483d843..9d6b4d5d87e24d72b29ab33ee805fe0d068cc30a 100644 --- a/tensorflow/contrib/input_pipeline/BUILD +++ b/tensorflow/contrib/input_pipeline/BUILD @@ -67,9 +67,9 @@ tf_custom_op_py_library( "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:state_ops", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/kernel_methods/BUILD b/tensorflow/contrib/kernel_methods/BUILD index ae1402b0e6688a0f43278999d1d93282ea2a11a5..a2f320ab11291e4049c8367e1f133a4fbcb72a62 100644 --- a/tensorflow/contrib/kernel_methods/BUILD +++ b/tensorflow/contrib/kernel_methods/BUILD @@ -64,6 +64,7 @@ py_test( name = "kernel_estimators_test", srcs = ["python/kernel_estimators_test.py"], srcs_version = "PY2AND3", + tags = ["notsan"], deps = [ ":kernel_methods", "//tensorflow/contrib/layers:layers_py", diff --git a/tensorflow/contrib/kfac/examples/convnet.py b/tensorflow/contrib/kfac/examples/convnet.py index a62780a9366590a678c93427c251e5b6fadccb1c..558bc294bc8ac129b3055ed46623c78a0d5a33e3 100644 --- a/tensorflow/contrib/kfac/examples/convnet.py +++ b/tensorflow/contrib/kfac/examples/convnet.py @@ -83,7 +83,7 @@ def conv_layer(layer_id, inputs, kernel_size, out_channels): activations = tf.nn.relu(preactivations) # layer.weights is a list. This converts it a (hashable) tuple. - return preactivations, activations, tuple(layer.weights) + return preactivations, activations, (layer.kernel, layer.bias) def max_pool_layer(layer_id, inputs, kernel_size, stride): @@ -128,7 +128,7 @@ def linear_layer(layer_id, inputs, output_size): return pre, params -def build_model(examples, labels, num_labels, num_ps_tasks=0): +def build_model(examples, labels, num_labels, layer_collection): """Builds a ConvNet classification model. Args: @@ -137,65 +137,64 @@ def build_model(examples, labels, num_labels, num_ps_tasks=0): labels: Tensor of shape [num_examples]. Contains integer IDs to be predicted by softmax for each example. num_labels: int. Number of distinct values 'labels' can take on. - num_ps_tasks: int. Number of parameter servers. If zero, variables - will be placed locally. + layer_collection: LayerCollection instance. Layers will be registered here. Returns: loss: 0-D Tensor representing loss to be minimized. - statistics: dict mapping strings to Tensors. Additional model evaluation - statistics. - layer_collection: LayerCollection instance describing model architecture. + accuracy: 0-D Tensor representing model's accuracy. """ - with tf.device(tf.train.replica_device_setter(num_ps_tasks)): - # Build a ConvNet. For each layer with parameters, we'll keep track of the - # preactivations, activations, weights, and bias. - tf.logging.info("Building model.") - pre0, act0, params0 = conv_layer( - layer_id=0, inputs=examples, kernel_size=5, out_channels=16) - act1 = max_pool_layer(layer_id=1, inputs=act0, kernel_size=3, stride=2) - pre2, act2, params2 = conv_layer( - layer_id=2, inputs=act1, kernel_size=5, out_channels=16) - act3 = max_pool_layer(layer_id=3, inputs=act2, kernel_size=3, stride=2) - flat_act3 = tf.reshape(act3, shape=[-1, int(np.prod(act3.shape[1:4]))]) - logits, params4 = linear_layer( - layer_id=4, inputs=flat_act3, output_size=num_labels) - loss = tf.reduce_mean( - tf.nn.sparse_softmax_cross_entropy_with_logits( - labels=labels, logits=logits)) - accuracy = tf.reduce_mean( - tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.float32)) - - tf.summary.scalar("loss", loss) - tf.summary.scalar("accuracy", accuracy) - - # Register parameters. K-FAC needs to know about the inputs, outputs, and - # parameters of each conv/fully connected layer and the logits powering the - # posterior probability over classes. - tf.logging.info("Building KFAC Optimizer.") - layer_collection = lc.LayerCollection() - layer_collection.register_conv2d(params0, (1, 1, 1, 1), "SAME", examples, - pre0) - layer_collection.register_conv2d(params2, (1, 1, 1, 1), "SAME", act1, pre2) - layer_collection.register_fully_connected(params4, flat_act3, logits) - layer_collection.register_categorical_predictive_distribution(logits) - - return loss, {"accuracy": accuracy}, layer_collection - - -def minimize_loss_single_machine(loss, statistics, layer_collection): + # Build a ConvNet. For each layer with parameters, we'll keep track of the + # preactivations, activations, weights, and bias. + tf.logging.info("Building model.") + pre0, act0, params0 = conv_layer( + layer_id=0, inputs=examples, kernel_size=5, out_channels=16) + act1 = max_pool_layer(layer_id=1, inputs=act0, kernel_size=3, stride=2) + pre2, act2, params2 = conv_layer( + layer_id=2, inputs=act1, kernel_size=5, out_channels=16) + act3 = max_pool_layer(layer_id=3, inputs=act2, kernel_size=3, stride=2) + flat_act3 = tf.reshape(act3, shape=[-1, int(np.prod(act3.shape[1:4]))]) + logits, params4 = linear_layer( + layer_id=4, inputs=flat_act3, output_size=num_labels) + loss = tf.reduce_mean( + tf.nn.sparse_softmax_cross_entropy_with_logits( + labels=labels, logits=logits)) + accuracy = tf.reduce_mean( + tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.float32)) + + tf.summary.scalar("loss", loss) + tf.summary.scalar("accuracy", accuracy) + + # Register parameters. K-FAC needs to know about the inputs, outputs, and + # parameters of each conv/fully connected layer and the logits powering the + # posterior probability over classes. + tf.logging.info("Building LayerCollection.") + layer_collection.register_conv2d(params0, (1, 1, 1, 1), "SAME", examples, + pre0) + layer_collection.register_conv2d(params2, (1, 1, 1, 1), "SAME", act1, pre2) + layer_collection.register_fully_connected(params4, flat_act3, logits) + layer_collection.register_categorical_predictive_distribution( + logits, name="logits") + + return loss, accuracy + + +def minimize_loss_single_machine(loss, + accuracy, + layer_collection, + session_config=None): """Minimize loss with K-FAC on a single machine. A single Session is responsible for running all of K-FAC's ops. Args: loss: 0-D Tensor. Loss to be minimized. - statistics: dict mapping strings to 0-D Tensors. Additional statistics to - run with each step. + accuracy: 0-D Tensor. Accuracy of classifier on current minibatch. layer_collection: LayerCollection instance describing model architecture. Used by K-FAC to construct preconditioner. + session_config: None or tf.ConfigProto. Configuration for tf.Session(). Returns: - final value for 'statistics'. + final value for 'accuracy'. """ # Train with K-FAC. global_step = tf.train.get_or_create_global_step() @@ -208,19 +207,19 @@ def minimize_loss_single_machine(loss, statistics, layer_collection): train_op = optimizer.minimize(loss, global_step=global_step) tf.logging.info("Starting training.") - with tf.train.MonitoredTrainingSession() as sess: + with tf.train.MonitoredTrainingSession(config=session_config) as sess: while not sess.should_stop(): - global_step_, loss_, statistics_, _, _ = sess.run( - [global_step, loss, statistics, train_op, optimizer.cov_update_op]) + global_step_, loss_, accuracy_, _, _ = sess.run( + [global_step, loss, accuracy, train_op, optimizer.cov_update_op]) if global_step_ % 100 == 0: sess.run(optimizer.inv_update_op) if global_step_ % 100 == 0: - tf.logging.info("global_step: %d | loss: %f | %s", global_step_, loss_, - statistics_) + tf.logging.info("global_step: %d | loss: %f | accuracy: %s", + global_step_, loss_, accuracy_) - return statistics_ + return accuracy_ def _is_gradient_task(task_id, num_tasks): @@ -252,8 +251,7 @@ def _num_gradient_tasks(num_tasks): def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master, - checkpoint_dir, loss, statistics, - layer_collection): + checkpoint_dir, loss, accuracy, layer_collection): """Minimize loss with an synchronous implementation of K-FAC. Different tasks are responsible for different parts of K-FAC's Ops. The first @@ -269,13 +267,13 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master, string to run locally. checkpoint_dir: string or None. Path to store checkpoints under. loss: 0-D Tensor. Loss to be minimized. - statistics: dict mapping strings to 0-D Tensors. Additional statistics to + accuracy: dict mapping strings to 0-D Tensors. Additional accuracy to run with each step. layer_collection: LayerCollection instance describing model architecture. Used by K-FAC to construct preconditioner. Returns: - final value for 'statistics'. + final value for 'accuracy'. Raises: ValueError: if task_id >= num_worker_tasks. @@ -318,12 +316,12 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master, else: raise ValueError("Which op should task %d do?" % task_id) - global_step_, loss_, statistics_, _ = sess.run( - [global_step, loss, statistics, learning_op]) - tf.logging.info("global_step: %d | loss: %f | %s", global_step_, loss_, - statistics_) + global_step_, loss_, accuracy_, _ = sess.run( + [global_step, loss, accuracy, learning_op]) + tf.logging.info("global_step: %d | loss: %f | accuracy: %s", global_step_, + loss_, accuracy_) - return statistics_ + return accuracy_ def train_mnist_single_machine(data_dir, num_epochs, use_fake_data=False): @@ -347,11 +345,69 @@ def train_mnist_single_machine(data_dir, num_epochs, use_fake_data=False): flatten_images=False) # Build a ConvNet. - loss, statistics, layer_collection = build_model( - examples, labels, num_labels=10) + layer_collection = lc.LayerCollection() + loss, accuracy = build_model( + examples, labels, num_labels=10, layer_collection=layer_collection) + + # Fit model. + return minimize_loss_single_machine(loss, accuracy, layer_collection) + + +def train_mnist_multitower(data_dir, num_epochs, num_towers, + use_fake_data=True): + """Train a ConvNet on MNIST. + + Args: + data_dir: string. Directory to read MNIST examples from. + num_epochs: int. Number of passes to make over the training set. + num_towers: int. Number of CPUs to split inference across. + use_fake_data: bool. If True, generate a synthetic dataset. + + Returns: + accuracy of model on the final minibatch of training data. + """ + # Load a dataset. + tf.logging.info("Loading MNIST into memory.") + tower_batch_size = 128 + batch_size = tower_batch_size * num_towers + tf.logging.info( + ("Loading MNIST into memory. Using batch_size = %d = %d towers * %d " + "tower batch size.") % (batch_size, num_towers, tower_batch_size)) + examples, labels = mnist.load_mnist( + data_dir, + num_epochs=num_epochs, + batch_size=batch_size, + use_fake_data=use_fake_data, + flatten_images=False) + + # Split minibatch across towers. + examples = tf.split(examples, num_towers) + labels = tf.split(labels, num_towers) + + # Build an MLP. Each tower's layers will be added to the LayerCollection. + layer_collection = lc.LayerCollection() + tower_results = [] + for tower_id in range(num_towers): + with tf.device("/cpu:%d" % tower_id): + with tf.name_scope("tower%d" % tower_id): + with tf.variable_scope(tf.get_variable_scope(), reuse=(tower_id > 0)): + tf.logging.info("Building tower %d." % tower_id) + tower_results.append( + build_model(examples[tower_id], labels[tower_id], 10, + layer_collection)) + losses, accuracies = zip(*tower_results) + + # Average across towers. + loss = tf.reduce_mean(losses) + accuracy = tf.reduce_mean(accuracies) # Fit model. - return minimize_loss_single_machine(loss, statistics, layer_collection) + session_config = tf.ConfigProto( + allow_soft_placement=False, device_count={ + "CPU": num_towers + }) + return minimize_loss_single_machine( + loss, accuracy, layer_collection, session_config=session_config) def train_mnist_distributed(task_id, @@ -385,13 +441,15 @@ def train_mnist_distributed(task_id, flatten_images=False) # Build a ConvNet. - loss, statistics, layer_collection = build_model( - examples, labels, num_labels=10, num_ps_tasks=num_ps_tasks) + layer_collection = lc.LayerCollection() + with tf.device(tf.train.replica_device_setter(num_ps_tasks)): + loss, accuracy = build_model( + examples, labels, num_labels=10, layer_collection=layer_collection) # Fit model. checkpoint_dir = None if data_dir is None else os.path.join(data_dir, "kfac") return minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, - master, checkpoint_dir, loss, statistics, + master, checkpoint_dir, loss, accuracy, layer_collection) diff --git a/tensorflow/contrib/kfac/examples/convnet_mnist_main.py b/tensorflow/contrib/kfac/examples/convnet_mnist_main.py index 2058c8b6bf4b0288acf85a3d731e22215ab33624..b0c6fbde198850c76af0bc1600dc23e926227229 100644 --- a/tensorflow/contrib/kfac/examples/convnet_mnist_main.py +++ b/tensorflow/contrib/kfac/examples/convnet_mnist_main.py @@ -33,7 +33,12 @@ FLAGS = None def main(argv): _ = argv - convnet.train_mnist_single_machine(FLAGS.data_dir, num_epochs=200) + + if FLAGS.num_towers > 1: + convnet.train_mnist_multitower( + FLAGS.data_dir, num_epochs=200, num_towers=FLAGS.num_towers) + else: + convnet.train_mnist_single_machine(FLAGS.data_dir, num_epochs=200) if __name__ == "__main__": @@ -43,5 +48,10 @@ if __name__ == "__main__": type=str, default="/tmp/mnist", help="Directory to store dataset in.") + parser.add_argument( + "--num_towers", + type=int, + default=1, + help="Number of CPUs to split minibatch across.") FLAGS, unparsed = parser.parse_known_args() tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/kfac/examples/mlp.py b/tensorflow/contrib/kfac/examples/mlp.py index ecebed2dd3ab717a91e63f45f25b3d97bc4e9e44..4275ceadc210ff471109b596e1c9aa260ce31ab5 100644 --- a/tensorflow/contrib/kfac/examples/mlp.py +++ b/tensorflow/contrib/kfac/examples/mlp.py @@ -32,6 +32,7 @@ opt = tf.contrib.kfac.optimizer __all__ = [ "fc_layer", "train_mnist", + "train_mnist_multitower", ] @@ -60,36 +61,30 @@ def fc_layer(layer_id, inputs, output_size): activations = tf.nn.tanh(preactivations) # layer.weights is a list. This converts it a (hashable) tuple. - return preactivations, activations, tuple(layer.weights) + return preactivations, activations, (layer.kernel, layer.bias) -def train_mnist(data_dir, num_epochs, use_fake_data=False): - """Train an MLP on MNIST. +def build_model(examples, labels, num_labels, layer_collection): + """Builds an MLP classification model. Args: - data_dir: string. Directory to read MNIST examples from. - num_epochs: int. Number of passes to make over the training set. - use_fake_data: bool. If True, generate a synthetic dataset. + examples: Tensor of shape [num_examples, num_features]. Represents inputs of + model. + labels: Tensor of shape [num_examples]. Contains integer IDs to be predicted + by softmax for each example. + num_labels: int. Number of distinct values 'labels' can take on. + layer_collection: LayerCollection instance describing model architecture. Returns: - accuracy of model on the final minibatch of training data. + loss: 0-D Tensor representing loss to be minimized. + accuracy: 0-D Tensor representing model's accuracy. """ - # Load a dataset. - tf.logging.info("Loading MNIST into memory.") - examples, labels = mnist.load_mnist( - data_dir, - num_epochs=num_epochs, - batch_size=64, - flatten_images=True, - use_fake_data=use_fake_data) - # Build an MLP. For each layer, we'll keep track of the preactivations, # activations, weights, and bias. - tf.logging.info("Building model.") pre0, act0, params0 = fc_layer(layer_id=0, inputs=examples, output_size=128) pre1, act1, params1 = fc_layer(layer_id=1, inputs=act0, output_size=64) pre2, act2, params2 = fc_layer(layer_id=2, inputs=act1, output_size=32) - logits, _, params3 = fc_layer(layer_id=3, inputs=act2, output_size=10) + logits, _, params3 = fc_layer(layer_id=3, inputs=act2, output_size=num_labels) loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( labels=labels, logits=logits)) @@ -99,16 +94,32 @@ def train_mnist(data_dir, num_epochs, use_fake_data=False): # Register parameters. K-FAC needs to know about the inputs, outputs, and # parameters of each layer and the logits powering the posterior probability # over classes. - tf.logging.info("Building KFAC Optimizer.") - layer_collection = lc.LayerCollection() + tf.logging.info("Building LayerCollection.") layer_collection.register_fully_connected(params0, examples, pre0) layer_collection.register_fully_connected(params1, act0, pre1) layer_collection.register_fully_connected(params2, act1, pre2) layer_collection.register_fully_connected(params3, act2, logits) - layer_collection.register_categorical_predictive_distribution(logits) + layer_collection.register_categorical_predictive_distribution( + logits, name="logits") + return loss, accuracy + + +def minimize(loss, accuracy, layer_collection, session_config=None): + """Minimize 'loss' with KfacOptimizer. + + Args: + loss: 0-D Tensor. Loss to be minimized. + accuracy: 0-D Tensor. Accuracy of classifier on current minibatch. + layer_collection: LayerCollection instance. Describes layers in model. + session_config: tf.ConfigProto. Configuration for tf.Session(). + + Returns: + accuracy of classifier on final minibatch. + """ # Train with K-FAC. We'll use a decreasing learning rate that's cut in 1/2 # every 10k iterations. + tf.logging.info("Building KFAC Optimizer.") global_step = tf.train.get_or_create_global_step() optimizer = opt.KfacOptimizer( learning_rate=tf.train.exponential_decay( @@ -120,7 +131,7 @@ def train_mnist(data_dir, num_epochs, use_fake_data=False): train_op = optimizer.minimize(loss, global_step=global_step) tf.logging.info("Starting training.") - with tf.train.MonitoredTrainingSession() as sess: + with tf.train.MonitoredTrainingSession(config=session_config) as sess: while not sess.should_stop(): # K-FAC has 3 primary ops, # - train_op: Update the weights with the minibatch's gradient. @@ -141,3 +152,90 @@ def train_mnist(data_dir, num_epochs, use_fake_data=False): global_step_, loss_, accuracy_) return accuracy_ + + +def train_mnist(data_dir, num_epochs, use_fake_data=False): + """Train an MLP on MNIST. + + Args: + data_dir: string. Directory to read MNIST examples from. + num_epochs: int. Number of passes to make over the training set. + use_fake_data: bool. If True, generate a synthetic dataset. + + Returns: + accuracy of model on the final minibatch of training data. + """ + # Load a dataset. + tf.logging.info("Loading MNIST into memory.") + examples, labels = mnist.load_mnist( + data_dir, + num_epochs=num_epochs, + batch_size=64, + flatten_images=True, + use_fake_data=use_fake_data) + + # Build an MLP. The model's layers will be added to the LayerCollection. + tf.logging.info("Building model.") + layer_collection = lc.LayerCollection() + loss, accuracy = build_model(examples, labels, 10, layer_collection) + + # Fit model. + minimize(loss, accuracy, layer_collection) + + +def train_mnist_multitower(data_dir, + num_epochs, + num_towers, + use_fake_data=False): + """Train an MLP on MNIST, splitting the minibatch across multiple towers. + + Args: + data_dir: string. Directory to read MNIST examples from. + num_epochs: int. Number of passes to make over the training set. + num_towers: int. Number of CPUs to split minibatch across. + use_fake_data: bool. If True, generate a synthetic dataset. + + Returns: + accuracy of model on the final minibatch of training data. + """ + # Load a dataset. + tower_batch_size = 64 + batch_size = tower_batch_size * num_towers + tf.logging.info( + ("Loading MNIST into memory. Using batch_size = %d = %d towers * %d " + "tower batch size.") % (batch_size, num_towers, tower_batch_size)) + examples, labels = mnist.load_mnist( + data_dir, + num_epochs=num_epochs, + batch_size=batch_size, + flatten_images=True, + use_fake_data=use_fake_data) + + # Split minibatch across towers. + examples = tf.split(examples, num_towers) + labels = tf.split(labels, num_towers) + + # Build an MLP. Each tower's layers will be added to the LayerCollection. + layer_collection = lc.LayerCollection() + tower_results = [] + for tower_id in range(num_towers): + with tf.device("/cpu:%d" % tower_id): + with tf.name_scope("tower%d" % tower_id): + with tf.variable_scope(tf.get_variable_scope(), reuse=(tower_id > 0)): + tf.logging.info("Building tower %d." % tower_id) + tower_results.append( + build_model(examples[tower_id], labels[tower_id], 10, + layer_collection)) + losses, accuracies = zip(*tower_results) + + # Average across towers. + loss = tf.reduce_mean(losses) + accuracy = tf.reduce_mean(accuracies) + + # Fit model. + session_config = tf.ConfigProto( + allow_soft_placement=False, device_count={ + "CPU": num_towers + }) + return minimize( + loss, accuracy, layer_collection, session_config=session_config) diff --git a/tensorflow/contrib/kfac/examples/mlp_mnist_main.py b/tensorflow/contrib/kfac/examples/mlp_mnist_main.py index a272f7d67a4195fbcfd83601272274afebec2406..b318c71a568be2d717745579df24134ceb3b6a0b 100644 --- a/tensorflow/contrib/kfac/examples/mlp_mnist_main.py +++ b/tensorflow/contrib/kfac/examples/mlp_mnist_main.py @@ -33,7 +33,11 @@ FLAGS = None def main(argv): _ = argv - mlp.train_mnist(FLAGS.data_dir, num_epochs=200) + if FLAGS.num_towers > 1: + mlp.train_mnist_multitower( + FLAGS.data_dir, num_epochs=200, num_towers=FLAGS.num_towers) + else: + mlp.train_mnist(FLAGS.data_dir, num_epochs=200) if __name__ == "__main__": @@ -43,5 +47,10 @@ if __name__ == "__main__": type=str, default="/tmp/mnist", help="Directory to store dataset in.") + parser.add_argument( + "--num_towers", + type=int, + default=1, + help="Number of CPUs to split minibatch across.") FLAGS, unparsed = parser.parse_known_args() tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) diff --git a/tensorflow/contrib/kfac/examples/tests/convnet_test.py b/tensorflow/contrib/kfac/examples/tests/convnet_test.py index b96dd227e19bf614c516e33bb98949591e99caf9..3c98c54ef6cbd527aa0035e0b6f40be961c6308d 100644 --- a/tensorflow/contrib/kfac/examples/tests/convnet_test.py +++ b/tensorflow/contrib/kfac/examples/tests/convnet_test.py @@ -66,8 +66,9 @@ class ConvNetTest(tf.test.TestCase): with tf.Graph().as_default(): x = tf.placeholder(tf.float32, [None, 6, 6, 3]) y = tf.placeholder(tf.int64, [None]) - loss, statistics, layer_collection = convnet.build_model( - x, y, num_labels=5) + layer_collection = lc.LayerCollection() + loss, accuracy = convnet.build_model( + x, y, num_labels=5, layer_collection=layer_collection) # Ensure layers and logits were registered. self.assertEqual(len(layer_collection.fisher_blocks), 3) @@ -80,7 +81,7 @@ class ConvNetTest(tf.test.TestCase): x: np.random.randn(10, 6, 6, 3).astype(np.float32), y: np.random.randint(5, size=10).astype(np.int64), } - sess.run([loss, statistics], feed_dict=feed_dict) + sess.run([loss, accuracy], feed_dict=feed_dict) def _build_toy_problem(self): """Construct a toy linear regression problem. @@ -90,8 +91,7 @@ class ConvNetTest(tf.test.TestCase): Returns: loss: 0-D Tensor representing loss to be minimized. - statistics: dict mapping strings to Tensors. Additional model evaluation - statistics. + accuracy: 0-D Tensors representing model accuracy. layer_collection: LayerCollection instance describing model architecture. """ x = np.asarray([[1.], [2.]]).astype(np.float32) @@ -101,34 +101,34 @@ class ConvNetTest(tf.test.TestCase): w = tf.get_variable("w", shape=[1, 1], initializer=tf.zeros_initializer()) y_hat = tf.matmul(x, w) loss = tf.reduce_mean(0.5 * tf.square(y_hat - y)) - statistics = {"loss": loss} + accuracy = loss layer_collection = lc.LayerCollection() layer_collection.register_fully_connected(params=w, inputs=x, outputs=y_hat) layer_collection.register_normal_predictive_distribution(y_hat) - return loss, statistics, layer_collection + return loss, accuracy, layer_collection def testMinimizeLossSingleMachine(self): with tf.Graph().as_default(): - loss, statistics, layer_collection = self._build_toy_problem() - statistics_ = convnet.minimize_loss_single_machine( - loss, statistics, layer_collection) - self.assertLess(statistics_["loss"], 1.0) + loss, accuracy, layer_collection = self._build_toy_problem() + accuracy_ = convnet.minimize_loss_single_machine(loss, accuracy, + layer_collection) + self.assertLess(accuracy_, 1.0) def testMinimizeLossDistributed(self): with tf.Graph().as_default(): - loss, statistics, layer_collection = self._build_toy_problem() - statistics_ = convnet.minimize_loss_distributed( + loss, accuracy, layer_collection = self._build_toy_problem() + accuracy_ = convnet.minimize_loss_distributed( task_id=0, num_worker_tasks=1, num_ps_tasks=0, master="", checkpoint_dir=None, loss=loss, - statistics=statistics, + accuracy=accuracy, layer_collection=layer_collection) - self.assertLess(statistics_["loss"], 1.0) + self.assertLess(accuracy_, 1.0) def testTrainMnistSingleMachine(self): with tf.Graph().as_default(): @@ -140,6 +140,12 @@ class ConvNetTest(tf.test.TestCase): convnet.train_mnist_single_machine( data_dir=None, num_epochs=1, use_fake_data=True) + def testTrainMnistMultitower(self): + with tf.Graph().as_default(): + # Ensure model training doesn't crash. + convnet.train_mnist_multitower( + data_dir=None, num_epochs=1, num_towers=2, use_fake_data=True) + def testTrainMnistDistributed(self): with tf.Graph().as_default(): # Ensure model training doesn't crash. diff --git a/tensorflow/contrib/kfac/examples/tests/mlp_test.py b/tensorflow/contrib/kfac/examples/tests/mlp_test.py index 833d02baed08a99117d563b8daaefceca6b8b286..34a942d27f64e2583c686c2ba3240bc636ed918b 100644 --- a/tensorflow/contrib/kfac/examples/tests/mlp_test.py +++ b/tensorflow/contrib/kfac/examples/tests/mlp_test.py @@ -47,6 +47,12 @@ class MlpTest(tf.test.TestCase): # but that takes a non-trivial amount of compute. mlp.train_mnist(data_dir=None, num_epochs=1, use_fake_data=True) + def testTrainMnistMultitower(self): + with tf.Graph().as_default(): + # Ensure model training doesn't crash. + mlp.train_mnist_multitower( + data_dir=None, num_epochs=1, num_towers=2, use_fake_data=True) + if __name__ == "__main__": tf.test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD index 1b2a5cdd3871f8a7848ee5a8df70452e58cc84a2..5d86373a232d55cd281d06cfc0606f4224d8f669 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD +++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD @@ -13,6 +13,8 @@ py_test( deps = [ "//tensorflow/contrib/kfac/python/ops:fisher_estimator", "//tensorflow/contrib/kfac/python/ops:layer_collection", + "//tensorflow/contrib/kfac/python/ops:utils", + "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", @@ -127,6 +129,20 @@ py_test( ], ) +py_test( + name = "loss_functions_test", + srcs = ["loss_functions_test.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/contrib/kfac/python/ops:loss_functions", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:framework_ops", + "//third_party/py/numpy", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py index 281274d88473ecd32bd18813a8a7e6a09d2dcc77..b52a7b52a7efd4292ad514c5a744c4da07082142 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py @@ -20,42 +20,80 @@ from __future__ import print_function from tensorflow.contrib.kfac.python.ops import estimator from tensorflow.contrib.kfac.python.ops import layer_collection as lc +from tensorflow.contrib.kfac.python.ops import utils from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test +_ALL_ESTIMATION_MODES = ["gradients", "empirical", "curvature_prop", "exact"] + class EstimatorTest(test.TestCase): - def testEstimatorInitManualRegistration(self): - with ops.Graph().as_default(): - layer_collection = lc.LayerCollection() + def setUp(self): + self._graph = ops.Graph() + with self._graph.as_default(): + self.layer_collection = lc.LayerCollection() - inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32) - weights = variable_scope.get_variable( - 'w', shape=(2, 2), dtype=dtypes.float32) - bias = variable_scope.get_variable( - 'b', initializer=init_ops.zeros_initializer(), shape=(2, 1)) - output = math_ops.matmul(inputs, weights) + bias + self.inputs = random_ops.random_normal((2, 2), dtype=dtypes.float32) + self.weights = variable_scope.get_variable( + "w", shape=(2, 2), dtype=dtypes.float32) + self.bias = variable_scope.get_variable( + "b", initializer=init_ops.zeros_initializer(), shape=(2, 1)) + self.output = math_ops.matmul(self.inputs, self.weights) + self.bias # Only register the weights. - layer_collection.register_fully_connected((weights,), inputs, output) + self.layer_collection.register_fully_connected( + params=(self.weights,), inputs=self.inputs, outputs=self.output) - outputs = math_ops.tanh(output) - layer_collection.register_categorical_predictive_distribution(outputs) + self.outputs = math_ops.tanh(self.output) + self.targets = array_ops.zeros_like(self.outputs) + self.layer_collection.register_categorical_predictive_distribution( + logits=self.outputs, targets=self.targets) + def testEstimatorInitManualRegistration(self): + with self._graph.as_default(): # We should be able to build an estimator for only the registered vars. - estimator.FisherEstimator([weights], 0.1, 0.2, layer_collection) + estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection) # Check that we throw an error if we try to build an estimator for vars # that were not manually registered. with self.assertRaises(ValueError): - estimator.FisherEstimator([weights, bias], 0.1, 0.2, layer_collection) + estimator.FisherEstimator([self.weights, self.bias], 0.1, 0.2, + self.layer_collection) + + # Check that we throw an error if we don't include registered variables, + # i.e. self.weights + with self.assertRaises(ValueError): + estimator.FisherEstimator([], 0.1, 0.2, self.layer_collection) + + @test.mock.patch.object(utils.SubGraph, "variable_uses", return_value=42) + def testVariableWrongNumberOfUses(self, mock_uses): + with self.assertRaises(ValueError): + estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection) + + def testInvalidEstimationMode(self): + with self.assertRaises(ValueError): + estimator.FisherEstimator([self.weights], 0.1, 0.2, self.layer_collection, + "not_a_real_mode") + + def testModeListCorrect(self): + with self._graph.as_default(): + est = estimator.FisherEstimator([self.weights], 0.1, 0.2, + self.layer_collection) + self.assertItemsEqual(_ALL_ESTIMATION_MODES, est._gradient_fns.keys()) + + def testAllModesBuild(self): + for mode in _ALL_ESTIMATION_MODES: + with self._graph.as_default(): + estimator.FisherEstimator([self.weights], 0.1, 0.2, + self.layer_collection, mode) -if __name__ == '__main__': +if __name__ == "__main__": test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py index f48d1980babe283d5bb6e911bdabc469481a74fb..dbf40fccc8257b1dec6cbd790adfa59161ab9049 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py @@ -209,6 +209,146 @@ class NaiveDiagonalFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +class FullyConnectedDiagonalFB(test.TestCase): + + def setUp(self): + super(FullyConnectedDiagonalFB, self).setUp() + + self.batch_size = 4 + self.input_size = 6 + self.output_size = 3 + + self.inputs = np.random.randn(self.batch_size, self.input_size).astype( + np.float32) + self.outputs = np.zeros([self.batch_size, self.output_size]).astype( + np.float32) + self.output_grads = np.random.randn(self.batch_size, + self.output_size).astype(np.float32) + self.w = np.random.randn(self.input_size, self.output_size).astype( + np.float32) + self.b = np.random.randn(self.output_size).astype(np.float32) + + def fisherApprox(self, has_bias=False): + """Fisher approximation using default inputs.""" + if has_bias: + inputs = np.concatenate( + [self.inputs, np.ones([self.batch_size, 1])], axis=1) + else: + inputs = self.inputs + return self.buildDiagonalFisherApproximation(inputs, self.output_grads) + + def buildDiagonalFisherApproximation(self, inputs, output_grads): + """Builds explicit diagonal Fisher approximation. + + Fisher's diagonal is (d loss / d w)'s elements squared for + d/dw = E[outer(input, output_grad)] + + where the expectation is taken over examples. + + Args: + inputs: np.array of shape [batch_size, input_size]. + output_grads: np.array of shape [batch_size, output_size]. + + Returns: + Diagonal np.array of shape [num_params, num_params] for num_params = + input_size * output_size. + """ + batch_size = inputs.shape[0] + assert output_grads.shape[0] == batch_size + input_size = inputs.shape[1] + output_size = output_grads.shape[1] + fisher_diag = np.zeros((input_size, output_size)) + for i in range(batch_size): + fisher_diag += np.square(np.outer(inputs[i], output_grads[i])) + return np.diag(fisher_diag.flatten()) / batch_size + + def testMultiply(self): + result, _ = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct Fisher-vector product. + expected_result = self.fisherApprox().dot(self.w.flatten()) + expected_result = expected_result.reshape( + [self.input_size, self.output_size]) + + self.assertAllClose(expected_result, result) + + def testMultiplyInverse(self): + _, result = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct inverse Fisher-vector product. + expected_result = np.linalg.inv(self.fisherApprox()).dot(self.w.flatten()) + expected_result = expected_result.reshape( + [self.input_size, self.output_size]) + + self.assertAllClose(expected_result, result) + + def testRegisterAdditionalMinibatch(self): + """Ensure 1 big minibatch and 2 small minibatches are equivalent.""" + multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps( + self.w, [self.inputs], [self.outputs], [self.output_grads]) + multiply_result_small, multiply_inverse_result_small = ( + self.runFisherBlockOps(self.w, + np.split(self.inputs, 2), + np.split(self.outputs, 2), + np.split(self.output_grads, 2))) + + self.assertAllClose(multiply_result_big, multiply_result_small) + self.assertAllClose(multiply_inverse_result_big, + multiply_inverse_result_small) + + def testMultiplyHasBias(self): + result, _ = self.runFisherBlockOps((self.w, self.b), [self.inputs], + [self.outputs], [self.output_grads]) + expected_result = self.fisherApprox(True).dot( + np.concatenate([self.w.flatten(), self.b.flatten()])) + expected_result = expected_result.reshape( + [self.input_size + 1, self.output_size]) + expected_result = (expected_result[:-1], expected_result[-1]) + + self.assertEqual(len(result), 2) + self.assertAllClose(expected_result[0], result[0]) + self.assertAllClose(expected_result[1], result[1]) + + def runFisherBlockOps(self, params, inputs, outputs, output_grads): + """Run Ops guaranteed by FisherBlock interface. + + Args: + params: Tensor or 2-tuple of Tensors. Represents weights or weights and + bias of this layer. + inputs: list of Tensors of shape [batch_size, input_size]. Inputs to + layer. + outputs: list of Tensors of shape [batch_size, output_size]. + Preactivations produced by layer. + output_grads: list of Tensors of shape [batch_size, output_size]. + Gradient of loss with respect to 'outputs'. + + Returns: + multiply_result: Result of FisherBlock.multiply(params) + multiply_inverse_result: Result of FisherBlock.multiply_inverse(params) + """ + with ops.Graph().as_default(), self.test_session() as sess: + inputs = as_tensors(inputs) + outputs = as_tensors(outputs) + output_grads = as_tensors(output_grads) + params = as_tensors(params) + + block = fb.FullyConnectedDiagonalFB( + lc.LayerCollection(), has_bias=isinstance(params, (tuple, list))) + for (i, o) in zip(inputs, outputs): + block.register_additional_minibatch(i, o) + + block.instantiate_factors((output_grads,), damping=0.0) + + sess.run(tf_variables.global_variables_initializer()) + sess.run(block._factor.make_covariance_update_op(0.0)) + multiply_result = sess.run(block.multiply(params)) + multiply_inverse_result = sess.run(block.multiply_inverse(params)) + + return multiply_result, multiply_inverse_result + + class FullyConnectedKFACBasicFBTest(test.TestCase): def testFullyConnectedKFACBasicFBInit(self): @@ -216,50 +356,51 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): random_seed.set_random_seed(200) inputs = array_ops.constant([1., 2.]) outputs = array_ops.constant([3., 4.]) - block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), inputs, - outputs) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection()) + block.register_additional_minibatch(inputs, outputs) - self.assertAllEqual(outputs, block.tensors_to_compute_grads()) + self.assertAllEqual([outputs], block.tensors_to_compute_grads()) def testInstantiateFactorsHasBias(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2.], [3., 4.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=True) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=True) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) def testInstantiateFactorsNoBias(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2.], [3., 4.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) def testMultiplyInverseTuple(self): with ops.Graph().as_default(), self.test_session() as sess: random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) sess.run(block._input_factor.make_inverse_update_ops()) sess.run(block._output_factor.make_inverse_update_ops()) - vector = (np.arange(2, 6).reshape(2, 2).astype(np.float32), np.arange( - 1, 3).reshape(2, 1).astype(np.float32)) + vector = ( + np.arange(2, 6).reshape(2, 2).astype(np.float32), # + np.arange(1, 3).reshape(2, 1).astype(np.float32)) output = block.multiply_inverse((array_ops.constant(vector[0]), array_ops.constant(vector[1]))) @@ -273,10 +414,10 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): random_seed.set_random_seed(200) inputs = array_ops.constant([[1., 2.], [3., 4.]]) outputs = array_ops.constant([[3., 4.], [5., 6.]]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -296,11 +437,11 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): inputs = array_ops.zeros([32, input_dim]) outputs = array_ops.zeros([32, output_dim]) params = array_ops.zeros([input_dim, output_dim]) - block = fb.FullyConnectedKFACBasicFB( - lc.LayerCollection(), inputs, outputs, has_bias=False) + block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False) + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 damping = 0. # This test is only valid without damping. - block.instantiate_factors((grads,), damping) + block.instantiate_factors(([grads],), damping) sess.run(state_ops.assign(block._input_factor._cov, _make_psd(3))) sess.run(state_ops.assign(block._output_factor._cov, _make_psd(2))) @@ -318,6 +459,188 @@ class FullyConnectedKFACBasicFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +class ConvDiagonalFBTest(test.TestCase): + + def setUp(self): + super(ConvDiagonalFBTest, self).setUp() + + self.batch_size = 2 + self.height = 8 + self.width = 4 + self.input_channels = 6 + self.output_channels = 3 + self.kernel_size = 1 + + self.inputs = np.random.randn(self.batch_size, self.height, self.width, + self.input_channels).astype(np.float32) + self.outputs = np.zeros( + [self.batch_size, self.height, self.width, + self.output_channels]).astype(np.float32) + self.output_grads = np.random.randn( + self.batch_size, self.height, self.width, self.output_channels).astype( + np.float32) + self.w = np.random.randn(self.kernel_size, self.kernel_size, + self.input_channels, self.output_channels).astype( + np.float32) + self.b = np.random.randn(self.output_channels).astype(np.float32) + + def fisherApprox(self, has_bias=False): + """Fisher approximation using default inputs.""" + if has_bias: + inputs = np.concatenate( + [self.inputs, + np.ones([self.batch_size, self.height, self.width, 1])], + axis=-1) + else: + inputs = self.inputs + return self.buildDiagonalFisherApproximation(inputs, self.output_grads, + self.kernel_size) + + def buildDiagonalFisherApproximation(self, inputs, output_grads, kernel_size): + r"""Builds explicit diagonal Fisher approximation. + + Fisher's diagonal is (d loss / d w)'s elements squared for + d/dw = E[\sum_{loc} outer(input_{loc}, output_grad_{loc})] + + where the expectation is taken over examples and the sum over (x, y) + locations upon which the convolution is applied. + + Args: + inputs: np.array of shape [batch_size, height, width, input_channels]. + output_grads: np.array of shape [batch_size, height, width, + output_channels]. + kernel_size: int. height and width of kernel. + + Returns: + Diagonal np.array of shape [num_params, num_params] for num_params = + kernel_size^2 * input_channels * output_channels. + """ + batch_size, height, width, input_channels = inputs.shape + assert output_grads.shape[0] == batch_size + assert output_grads.shape[1] == height + assert output_grads.shape[2] == width + output_channels = output_grads.shape[3] + + # If kernel_size == 1, then we don't need to worry about capturing context + # around the pixel upon which a convolution is applied. This makes testing + # easier. + assert kernel_size == 1, "kernel_size != 1 isn't supported." + num_locations = height * width + inputs = np.reshape(inputs, [batch_size, num_locations, input_channels]) + output_grads = np.reshape(output_grads, + [batch_size, num_locations, output_channels]) + + fisher_diag = np.zeros((input_channels, output_channels)) + for i in range(batch_size): + # Each example's approximation is a square(sum-of-outer-products). + example_fisher_diag = np.zeros((input_channels, output_channels)) + for j in range(num_locations): + example_fisher_diag += np.outer(inputs[i, j], output_grads[i, j]) + fisher_diag += np.square(example_fisher_diag) + + # Normalize by batch_size (not num_locations). + return np.diag(fisher_diag.flatten()) / batch_size + + def testMultiply(self): + result, _ = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct Fisher-vector product. + expected_result = self.fisherApprox().dot(self.w.flatten()) + expected_result = expected_result.reshape([ + self.kernel_size, self.kernel_size, self.input_channels, + self.output_channels + ]) + + self.assertAllClose(expected_result, result) + + def testMultiplyInverse(self): + _, result = self.runFisherBlockOps(self.w, [self.inputs], [self.outputs], + [self.output_grads]) + + # Construct inverse Fisher-vector product. + expected_result = np.linalg.inv(self.fisherApprox()).dot(self.w.flatten()) + expected_result = expected_result.reshape([ + self.kernel_size, self.kernel_size, self.input_channels, + self.output_channels + ]) + + self.assertAllClose(expected_result, result, atol=1e-3) + + def testRegisterAdditionalMinibatch(self): + """Ensure 1 big minibatch and 2 small minibatches are equivalent.""" + multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps( + self.w, [self.inputs], [self.outputs], [self.output_grads]) + multiply_result_small, multiply_inverse_result_small = ( + self.runFisherBlockOps(self.w, + np.split(self.inputs, 2), + np.split(self.outputs, 2), + np.split(self.output_grads, 2))) + + self.assertAllClose(multiply_result_big, multiply_result_small) + self.assertAllClose(multiply_inverse_result_big, + multiply_inverse_result_small) + + def testMultiplyHasBias(self): + result, _ = self.runFisherBlockOps((self.w, self.b), [self.inputs], + [self.outputs], [self.output_grads]) + # Clone 'b' along 'input_channels' dimension. + b_filter = np.tile( + np.reshape(self.b, [1, 1, 1, self.output_channels]), + [self.kernel_size, self.kernel_size, 1, 1]) + params = np.concatenate([self.w, b_filter], axis=2) + expected_result = self.fisherApprox(True).dot(params.flatten()) + + # Extract 'b' from concatenated parameters. + expected_result = expected_result.reshape([ + self.kernel_size, self.kernel_size, self.input_channels + 1, + self.output_channels + ]) + expected_result = (expected_result[:, :, 0:-1, :], np.reshape( + expected_result[:, :, -1, :], [self.output_channels])) + + self.assertEqual(len(result), 2) + self.assertAllClose(expected_result[0], result[0]) + self.assertAllClose(expected_result[1], result[1]) + + def runFisherBlockOps(self, params, inputs, outputs, output_grads): + """Run Ops guaranteed by FisherBlock interface. + + Args: + params: Tensor or 2-tuple of Tensors. Represents weights or weights and + bias of this layer. + inputs: list of Tensors of shape [batch_size, input_size]. Inputs to + layer. + outputs: list of Tensors of shape [batch_size, output_size]. + Preactivations produced by layer. + output_grads: list of Tensors of shape [batch_size, output_size]. + Gradient of loss with respect to 'outputs'. + + Returns: + multiply_result: Result of FisherBlock.multiply(params) + multiply_inverse_result: Result of FisherBlock.multiply_inverse(params) + """ + with ops.Graph().as_default(), self.test_session() as sess: + inputs = as_tensors(inputs) + outputs = as_tensors(outputs) + output_grads = as_tensors(output_grads) + params = as_tensors(params) + + block = fb.ConvDiagonalFB( + lc.LayerCollection(), params, strides=[1, 1, 1, 1], padding='SAME') + for (i, o) in zip(inputs, outputs): + block.register_additional_minibatch(i, o) + + block.instantiate_factors((output_grads,), damping=0.0) + + sess.run(tf_variables.global_variables_initializer()) + sess.run(block._factor.make_covariance_update_op(0.0)) + multiply_result = sess.run(block.multiply(params)) + multiply_inverse_result = sess.run(block.multiply_inverse(params)) + + return multiply_result, multiply_inverse_result + + class ConvKFCBasicFBTest(test.TestCase): def _testConvKFCBasicFBInitParams(self, params): @@ -329,10 +652,10 @@ class ConvKFCBasicFBTest(test.TestCase): params = array_ops.constant(params) inputs = random_ops.random_normal((2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - [1, 1, 1], 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, [1, 1, 1], 'SAME') + block.register_additional_minibatch(inputs, outputs) - self.assertAllEqual(outputs, block.tensors_to_compute_grads()) + self.assertAllEqual([outputs], block.tensors_to_compute_grads()) def testConvKFCBasicFBInitParamsParamsTuple(self): self._testConvKFCBasicFBInitParams([np.array([1., 2.]), np.array(3.)]) @@ -346,10 +669,11 @@ class ConvKFCBasicFBTest(test.TestCase): params = random_ops.random_normal((2, 2, 2, 2)) inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -371,11 +695,12 @@ class ConvKFCBasicFBTest(test.TestCase): params = random_ops.random_normal((2, 2, 2, 2)) inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) self.assertFalse(block._has_bias) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -393,11 +718,12 @@ class ConvKFCBasicFBTest(test.TestCase): params = [random_ops.random_normal((2, 2, 2, 2))] inputs = random_ops.random_normal((2, 2, 2, 2)) outputs = random_ops.random_normal((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) self.assertTrue(block._has_bias) grads = outputs**2 - block.instantiate_factors((grads,), 0.5) + block.instantiate_factors(([grads],), 0.5) # Make sure our inverse is something other than the identity. sess.run(tf_variables.global_variables_initializer()) @@ -415,11 +741,12 @@ class ConvKFCBasicFBTest(test.TestCase): params = array_ops.zeros((2, 2, 2, 2)) inputs = array_ops.zeros((2, 2, 2, 2)) outputs = array_ops.zeros((2, 2, 2, 2)) - block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, inputs, outputs, - (1, 1, 1, 1), 'SAME') + block = fb.ConvKFCBasicFB(lc.LayerCollection(), params, (1, 1, 1, 1), + 'SAME') + block.register_additional_minibatch(inputs, outputs) grads = outputs**2 damping = 0. # This test is only valid without damping. - block.instantiate_factors((grads,), damping) + block.instantiate_factors(([grads],), damping) sess.run(state_ops.assign(block._input_factor._cov, _make_psd(8))) sess.run(state_ops.assign(block._output_factor._cov, _make_psd(2))) @@ -437,5 +764,11 @@ class ConvKFCBasicFBTest(test.TestCase): self.assertAllClose(output_flat, explicit) +def as_tensors(tensor_or_tuple): + """Converts a potentially nested tuple of np.array to Tensors.""" + if isinstance(tensor_or_tuple, (tuple, list)): + return tuple(as_tensors(t) for t in tensor_or_tuple) + return ops.convert_to_tensor(tensor_or_tuple) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py index 633104ace01dda6a6ba1ba058486ba39f18326e7..db7ab63c7d1166649acbe41851a5876d8af476db 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py @@ -30,6 +30,43 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.platform import test +class LayerParametersDictTest(test.TestCase): + + def testSetItem(self): + """Ensure insertion, contains, retrieval works for supported key types.""" + with ops.Graph().as_default(): + lp_dict = layer_collection.LayerParametersDict() + + x = array_ops.constant(0) + y0 = array_ops.constant(0) + y1 = array_ops.constant(0) + z0 = array_ops.constant(0) + z1 = array_ops.constant(0) + keys = [x, (y0, y1), [z0, z1]] + for key in keys: + lp_dict[key] = key + + for key in keys: + self.assertTrue(key in lp_dict) + self.assertEqual(lp_dict[key], key) + + def testSetItemOverlap(self): + """Ensure insertion fails if key overlaps with existing key.""" + with ops.Graph().as_default(): + lp_dict = layer_collection.LayerParametersDict() + + x = array_ops.constant(0) + y = array_ops.constant(0) + lp_dict[x] = 'value' + + with self.assertRaises(ValueError): + lp_dict[(x, y)] = 'value' + + # Ensure 'y' wasn't inserted. + self.assertTrue(x in lp_dict) + self.assertFalse(y in lp_dict) + + class LayerCollectionTest(test.TestCase): def testLayerCollectionInit(self): @@ -44,9 +81,18 @@ class LayerCollectionTest(test.TestCase): lc = layer_collection.LayerCollection() lc.register_fully_connected( array_ops.constant(1), array_ops.constant(2), array_ops.constant(3)) + lc.register_fully_connected( + array_ops.constant(1), + array_ops.constant(2), + array_ops.constant(3), + approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_conv2d( array_ops.constant(4), [1, 1, 1, 1], 'SAME', array_ops.ones((1, 1, 1, 1)), array_ops.constant(3)) + lc.register_conv2d( + array_ops.constant(4), [1, 1, 1, 1], 'SAME', + array_ops.ones((1, 1, 1, 1)), array_ops.constant(3), + approx=layer_collection.APPROX_DIAGONAL_NAME) lc.register_generic( array_ops.constant(5), 16, approx=layer_collection.APPROX_FULL_NAME) lc.register_generic( @@ -54,7 +100,7 @@ class LayerCollectionTest(test.TestCase): 16, approx=layer_collection.APPROX_DIAGONAL_NAME) - self.assertEqual(4, len(lc.get_blocks())) + self.assertEqual(6, len(lc.get_blocks())) def testRegisterBlocksMultipleRegistrations(self): with ops.Graph().as_default(): @@ -157,6 +203,83 @@ class LayerCollectionTest(test.TestCase): double_loss = sess.run(lc2.total_sampled_loss()) self.assertAlmostEqual(2 * single_loss, double_loss) + def testLossFunctionByName(self): + """Ensure loss functions can be identified by name.""" + with ops.Graph().as_default(): + logits = linalg_ops.eye(2) + lc = layer_collection.LayerCollection() + + # Create a new loss function by name. + lc.register_categorical_predictive_distribution(logits, name='loss1') + self.assertEqual(1, len(lc.losses)) + + # Add logits to same loss function. + lc.register_categorical_predictive_distribution( + logits, name='loss1', reuse=True) + self.assertEqual(1, len(lc.losses)) + + # Add another new loss function. + lc.register_categorical_predictive_distribution(logits, name='loss2') + self.assertEqual(2, len(lc.losses)) + + def testLossFunctionWithoutName(self): + """Ensure loss functions get unique names if 'name' not specified.""" + with ops.Graph().as_default(): + logits = linalg_ops.eye(2) + lc = layer_collection.LayerCollection() + + # Create a new loss function with default names. + lc.register_categorical_predictive_distribution(logits) + lc.register_categorical_predictive_distribution(logits) + self.assertEqual(2, len(lc.losses)) + + def testCategoricalPredictiveDistributionMultipleMinibatches(self): + """Ensure multiple minibatches are registered.""" + with ops.Graph().as_default(): + batch_size = 3 + output_size = 2 + logits = array_ops.zeros([batch_size, output_size]) + targets = array_ops.ones([batch_size], dtype=dtypes.int32) + lc = layer_collection.LayerCollection() + + # Create a new loss function. + lc.register_categorical_predictive_distribution( + logits, targets=targets, name='loss1') + + # Can add when reuse=True + lc.register_categorical_predictive_distribution( + logits, targets=targets, name='loss1', reuse=True) + + # Can add when reuse=VARIABLE_SCOPE and reuse=True there. + with variable_scope.variable_scope( + variable_scope.get_variable_scope(), reuse=True): + lc.register_categorical_predictive_distribution( + logits, + targets=targets, + name='loss1', + reuse=layer_collection.VARIABLE_SCOPE) + + # Can't add when reuse=False + with self.assertRaises(KeyError): + lc.register_categorical_predictive_distribution( + logits, targets=targets, name='loss1', reuse=False) + + # Can't add when reuse=VARIABLE_SCOPE and reuse=False there. + with self.assertRaises(KeyError): + lc.register_categorical_predictive_distribution( + logits, + targets=targets, + name='loss1', + reuse=layer_collection.VARIABLE_SCOPE) + + self.assertEqual(len(lc.losses), 1) + loss = lc.losses[0] + + # Three successful registrations. + self.assertEqual(loss.params.shape.as_list(), + [3 * batch_size, output_size]) + self.assertEqual(loss.targets.shape.as_list(), [3 * batch_size]) + def testRegisterCategoricalPredictiveDistributionBatchSize1(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) @@ -206,6 +329,83 @@ class LayerCollectionTest(test.TestCase): single_loss = sess.run(lc.total_loss()) self.assertAlmostEqual(7.6983433, single_loss) + def ensureLayerReuseWorks(self, register_fn): + """Ensure the 'reuse' keyword argument function as intended. + + Args: + register_fn: function for registering a layer. Arguments are + layer_collection, reuse, and approx. + """ + # Fails on second if reuse=False. + lc = layer_collection.LayerCollection() + register_fn(lc) + with self.assertRaises(ValueError): + register_fn(lc, reuse=False) + + # Succeeds on second if reuse=True. + lc = layer_collection.LayerCollection() + register_fn(lc) + register_fn(lc, reuse=True) + + # Fails on second if reuse=VARIABLE_SCOPE and no variable reuse. + lc = layer_collection.LayerCollection() + register_fn(lc) + with self.assertRaises(ValueError): + register_fn(lc, reuse=layer_collection.VARIABLE_SCOPE) + + # Succeeds on second if reuse=VARIABLE_SCOPE and variable reuse. + lc = layer_collection.LayerCollection() + register_fn(lc) + with variable_scope.variable_scope( + variable_scope.get_variable_scope(), reuse=True): + register_fn(lc, reuse=layer_collection.VARIABLE_SCOPE) + + # Fails if block type changes. + lc = layer_collection.LayerCollection() + register_fn(lc, approx=layer_collection.APPROX_KRONECKER_NAME) + with self.assertRaises(ValueError): + register_fn(lc, approx=layer_collection.APPROX_DIAGONAL_NAME, reuse=True) + + # Fails if reuse requested but no FisherBlock exists. + lc = layer_collection.LayerCollection() + with self.assertRaises(KeyError): + register_fn(lc, reuse=True) + + def testRegisterFullyConnectedReuse(self): + """Ensure the 'reuse' works with register_fully_connected.""" + with ops.Graph().as_default(): + inputs = array_ops.ones([2, 10]) + outputs = array_ops.zeros([2, 5]) + params = ( + variable_scope.get_variable('w', [10, 5]), # + variable_scope.get_variable('b', [5])) + + def register_fn(lc, **kwargs): + lc.register_fully_connected( + params=params, inputs=inputs, outputs=outputs, **kwargs) + + self.ensureLayerReuseWorks(register_fn) + + def testRegisterConv2dReuse(self): + """Ensure the 'reuse' works with register_conv2d.""" + with ops.Graph().as_default(): + inputs = array_ops.ones([2, 5, 5, 10]) + outputs = array_ops.zeros([2, 5, 5, 3]) + params = ( + variable_scope.get_variable('w', [1, 1, 10, 3]), # + variable_scope.get_variable('b', [3])) + + def register_fn(lc, **kwargs): + lc.register_conv2d( + params=params, + strides=[1, 1, 1, 1], + padding='SAME', + inputs=inputs, + outputs=outputs, + **kwargs) + + self.ensureLayerReuseWorks(register_fn) + def testMakeOrGetFactor(self): with ops.Graph().as_default(): random_seed.set_random_seed(200) @@ -237,10 +437,20 @@ class LayerCollectionTest(test.TestCase): self.assertTrue(all([var.name.startswith(scope) for var in variables])) def testGetUseCountMap(self): + """Ensure get_use_count_map() sums 'num_registered_minibatches'.""" + + class MockFisherBlock(object): + + num_registered_minibatches = 2 + lc = layer_collection.LayerCollection() - lc.fisher_blocks = {'a': 1, ('a', 'c'): 2, ('b', 'c'): 2} + lc.fisher_blocks = { + 'a': MockFisherBlock(), + ('a', 'c'): MockFisherBlock(), + ('b', 'c'): MockFisherBlock() + } use_count_map = lc.get_use_count_map() - self.assertDictEqual({'a': 2, 'b': 1, 'c': 2}, use_count_map) + self.assertDictEqual({'a': 4, 'b': 2, 'c': 4}, use_count_map) if __name__ == '__main__': diff --git a/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py new file mode 100644 index 0000000000000000000000000000000000000000..87339cb059802ec8944d5d1ae4557ee34550cd60 --- /dev/null +++ b/tensorflow/contrib/kfac/python/kernel_tests/loss_functions_test.py @@ -0,0 +1,101 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tf.contrib.kfac.loss_functions.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.contrib.kfac.python.ops import loss_functions +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import ops +from tensorflow.python.ops import array_ops +from tensorflow.python.platform import test + + +class InsertSliceInZerosTest(test.TestCase): + + def testBadShape(self): + bad_shaped_ones = array_ops.ones(shape=[1, 3]) # n.b. shape[1] != 1 + with self.assertRaises(ValueError): + loss_functions.insert_slice_in_zeros(bad_shaped_ones, 1, 42, 17) + + def test3d(self): + input_tensor = constant_op.constant([[[1, 2]], [[3, 4]]]) + expected_output_array = [[[1, 2], [0, 0]], [[3, 4], [0, 0]]] + op = loss_functions.insert_slice_in_zeros(input_tensor, 1, 2, 0) + with self.test_session() as sess: + actual_output_array = sess.run(op) + self.assertAllEqual(expected_output_array, actual_output_array) + + +class CategoricalLogitsNegativeLogProbLossTest(test.TestCase): + + def testSample(self): + """Ensure samples can be drawn.""" + with ops.Graph().as_default(), self.test_session() as sess: + logits = np.asarray([ + [0., 0., 0.], # + [1., -1., 0.] + ]).astype(np.float32) + loss = loss_functions.CategoricalLogitsNegativeLogProbLoss( + array_ops.constant(logits)) + sample = loss.sample(42) + sample = sess.run(sample) + self.assertEqual(sample.shape, (2,)) + + def testEvaluateOnTargets(self): + """Ensure log probability can be evaluated correctly.""" + with ops.Graph().as_default(), self.test_session() as sess: + logits = np.asarray([ + [0., 0., 0.], # + [1., -1., 0.] + ]).astype(np.float32) + targets = np.asarray([2, 1]).astype(np.int32) + loss = loss_functions.CategoricalLogitsNegativeLogProbLoss( + array_ops.constant(logits), targets=array_ops.constant(targets)) + neg_log_prob = loss.evaluate() + neg_log_prob = sess.run(neg_log_prob) + + # Calculate explicit log probability of targets. + probs = np.exp(logits) / np.sum(np.exp(logits), axis=1, keepdims=True) + log_probs = np.log([ + probs[0, targets[0]], # + probs[1, targets[1]] + ]) + expected_log_prob = np.sum(log_probs) + + self.assertAllClose(neg_log_prob, -expected_log_prob) + + def testEvaluateOnSample(self): + """Ensure log probability of a sample can be drawn.""" + with ops.Graph().as_default(), self.test_session() as sess: + logits = np.asarray([ + [0., 0., 0.], # + [1., -1., 0.] + ]).astype(np.float32) + loss = loss_functions.CategoricalLogitsNegativeLogProbLoss( + array_ops.constant(logits)) + neg_log_prob = loss.evaluate_on_sample(42) + + # Simply ensure this doesn't crash. As the output is random, it's + # difficult to say if the output is correct or not... + neg_log_prob = sess.run(neg_log_prob) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py index 5f28f57f6a37074b40fddd690c71292b785490b6..9325aa1b7325fa9cf546d66e6505affa1af7db4d 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/optimizer_test.py @@ -21,7 +21,6 @@ from __future__ import print_function import numpy as np from tensorflow.contrib.kfac.python.ops import layer_collection as lc -from tensorflow.contrib.kfac.python.ops import loss_functions as lf from tensorflow.contrib.kfac.python.ops import optimizer from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -124,9 +123,8 @@ class OptimizerTest(test.TestCase): def testUpdateVelocities(self): with ops.Graph().as_default(), self.test_session() as sess: layers = lc.LayerCollection() - layers.losses = [ - lf.CategoricalLogitsNegativeLogProbLoss(array_ops.constant([1.0])) - ] + layers.register_categorical_predictive_distribution( + array_ops.constant([1.0])) opt = optimizer.KfacOptimizer( 0.1, 0.2, 0.3, layers, momentum=0.5, momentum_type='regular') x = variable_scope.get_variable('x', initializer=array_ops.ones((2, 2))) diff --git a/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py b/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py index 779a8179bb07303ff43eba064763c20b9be71dbe..55fe38e3e9aab2dbd70a45cdc8fa0c208b036db0 100644 --- a/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py +++ b/tensorflow/contrib/kfac/python/kernel_tests/utils_test.py @@ -63,6 +63,39 @@ class SequenceDictTest(test.TestCase): self.assertItemsEqual(list(zip(keys, values)), seq_dict.items()) +class SubGraphTest(test.TestCase): + + def testBasicGraph(self): + a = array_ops.constant([[1., 2.], [3., 4.]]) + b = array_ops.constant([[5., 6.], [7., 8.]]) + c = a + b + d = a * b + sub_graph = utils.SubGraph((c,)) + self.assertTrue(sub_graph.is_member(a)) + self.assertTrue(sub_graph.is_member(b)) + self.assertTrue(sub_graph.is_member(c)) + self.assertFalse(sub_graph.is_member(d)) + + def testRepeatedAdds(self): + a = array_ops.constant([[1., 2.], [3., 4.]]) + b = array_ops.constant([[5., 6.], [7., 8.]]) + c = a + b + a # note that a appears twice in this graph + sub_graph = utils.SubGraph((c,)) + self.assertTrue(sub_graph.is_member(a)) + self.assertTrue(sub_graph.is_member(b)) + self.assertTrue(sub_graph.is_member(c)) + + def testFilterList(self): + a = array_ops.constant([[1., 2.], [3., 4.]]) + b = array_ops.constant([[5., 6.], [7., 8.]]) + c = a + b + d = a * b + sub_graph = utils.SubGraph((c,)) + input_list = [b, d] + filtered_list = sub_graph.filter_list(input_list) + self.assertEqual(filtered_list, [b]) + + class UtilsTest(test.TestCase): def _fully_connected_layer_params(self): diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD index 8b82f6e3147efbc204320f7be631448443287b1b..de4b8920b849dbf2117657de6e7c26f94f4d0363 100644 --- a/tensorflow/contrib/kfac/python/ops/BUILD +++ b/tensorflow/contrib/kfac/python/ops/BUILD @@ -66,6 +66,7 @@ py_library( deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", "//tensorflow/python/ops/distributions", "@six_archive//:six", ], @@ -89,6 +90,7 @@ py_library( ":utils", "//tensorflow/python:gradients", "//tensorflow/python:math_ops", + "//tensorflow/python:util", ], ) @@ -113,7 +115,9 @@ py_library( "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", + "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py b/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py index bf59a92fa677810dad62c49e8085d1a8202b7fa0..21b5cde9b931a95110c9a5fd7930a3a4ee74b207 100644 --- a/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py +++ b/tensorflow/contrib/kfac/python/ops/curvature_matrix_vector_products.py @@ -36,13 +36,13 @@ class CurvatureMatrixVectorProductComputer(object): For example, the Fisher associated with a log-prob loss w.r.t. the parameters. - The vecs argument to each method are lists of tensors that must be the + The 'vecs' argument to each method are lists of tensors that must be the size as the corresponding ones from "wrt_tensors". They represent the vector being multiplied. "factors" of the matrix M are defined as matrices B such that B*B^T = M. - Methods that multiply by the factor B take a "loss_inner_vecs" argument - instead of vecs, which must be a list of tensors with shapes given by the + Methods that multiply by the factor B take a 'loss_inner_vecs' argument + instead of 'vecs', which must be a list of tensors with shapes given by the corresponding XXX_inner_shapes property. Note that matrix-vector products are not normalized by the batch size, nor @@ -61,7 +61,8 @@ class CurvatureMatrixVectorProductComputer(object): Args: losses: A list of LossFunction instances whose sum defines the total loss. wrt_tensors: A list of Tensors to compute the differential quantities - defining the matrices with respect to (see class description). + (defining the matrices) with respect to. See class description for more + info. """ self._losses = losses self._inputs_to_losses = list(loss.inputs for loss in losses) @@ -73,24 +74,23 @@ class CurvatureMatrixVectorProductComputer(object): return math_ops.add_n(tuple(loss.evaluate() for loss in self._losses)) # Jacobian multiplication functions: - # NOTE: These implementations use tf.gradients and thus aren't actually - # computing partial derivatives, but total derivatives instead (despite what - # the documentation for tf.gradients says). Because we require partial - # derivatives for Jacobians this implementation will only be correct if the - # partial derivatives are equal to the full derivatives. This happens as long - # as the elements of wrt_tensors don't depend on each other in the graph. If - # these tensors are standard neural network parameters this will be true. def _multiply_jacobian(self, vecs): """Multiply vecs by the Jacobian of losses.""" + # We stop gradients at wrt_tensors to produce partial derivatives (which is + # what we want for Jacobians). jacobian_vecs_flat = utils.fwd_gradients( - self._inputs_to_losses_flat, self._wrt_tensors, grad_xs=vecs) + self._inputs_to_losses_flat, self._wrt_tensors, grad_xs=vecs, + stop_gradients=self._wrt_tensors) return nest.pack_sequence_as(self._inputs_to_losses, jacobian_vecs_flat) def _multiply_jacobian_transpose(self, loss_vecs): """Multiply vecs by the transpose Jacobian of losses.""" loss_vecs_flat = nest.flatten(loss_vecs) + # We stop gradients at wrt_tensors to produce partial derivatives (which is + # what we want for Jacobians). return gradients_impl.gradients( - self._inputs_to_losses_flat, self._wrt_tensors, grad_ys=loss_vecs_flat) + self._inputs_to_losses_flat, self._wrt_tensors, grad_ys=loss_vecs_flat, + stop_gradients=self._wrt_tensors) # Losses Fisher/Hessian multiplication functions: def _multiply_loss_fisher(self, loss_vecs): diff --git a/tensorflow/contrib/kfac/python/ops/estimator.py b/tensorflow/contrib/kfac/python/ops/estimator.py index c81086416c52d6ed828a8a8fda47a405124ff2b5..6e2c9ecdce7ad9f98a5beb016770ad2b1e197b0a 100644 --- a/tensorflow/contrib/kfac/python/ops/estimator.py +++ b/tensorflow/contrib/kfac/python/ops/estimator.py @@ -80,6 +80,12 @@ class FisherEstimator(object): self._layers = layer_collection self._layers.create_subgraph() self._check_registration(variables) + self._gradient_fns = { + "gradients": self._get_grads_lists_gradients, + "empirical": self._get_grads_lists_empirical, + "curvature_prop": self._get_grads_lists_curvature_prop, + "exact": self._get_grads_lists_exact + } setup = self._setup(cov_ema_decay) self.cov_update_op, self.inv_update_op, self.inv_updates_dict = setup @@ -201,75 +207,73 @@ class FisherEstimator(object): Raises: ValueError: If estimation_mode was improperly specified at construction. """ - damping = self.damping - fisher_blocks_list = self._layers.get_blocks() - tensors_to_compute_grads = [ fb.tensors_to_compute_grads() for fb in fisher_blocks_list ] - tensors_to_compute_grads_flat = nest.flatten(tensors_to_compute_grads) - - if self._estimation_mode == "gradients": - grads_flat = gradients_impl.gradients(self._layers.total_sampled_loss(), - tensors_to_compute_grads_flat) - grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat) - grads_lists = tuple((grad,) for grad in grads_all) - - elif self._estimation_mode == "empirical": - grads_flat = gradients_impl.gradients(self._layers.total_loss(), - tensors_to_compute_grads_flat) - grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat) - grads_lists = tuple((grad,) for grad in grads_all) - - elif self._estimation_mode == "curvature_prop": - loss_inputs = list(loss.inputs for loss in self._layers.losses) - loss_inputs_flat = nest.flatten(loss_inputs) - - transformed_random_signs = list(loss.multiply_fisher_factor( - utils.generate_random_signs(loss.fisher_factor_inner_shape)) - for loss in self._layers.losses) - - transformed_random_signs_flat = nest.flatten(transformed_random_signs) - - grads_flat = gradients_impl.gradients(loss_inputs_flat, - tensors_to_compute_grads_flat, - grad_ys - =transformed_random_signs_flat) - grads_all = nest.pack_sequence_as(tensors_to_compute_grads, grads_flat) - grads_lists = tuple((grad,) for grad in grads_all) - - elif self._estimation_mode == "exact": - # Loop over all coordinates of all losses. - grads_all = [] - for loss in self._layers.losses: - for index in np.ndindex(*loss.fisher_factor_inner_static_shape[1:]): - transformed_one_hot = loss.multiply_fisher_factor_replicated_one_hot( - index) - grads_flat = gradients_impl.gradients(loss.inputs, - tensors_to_compute_grads_flat, - grad_ys=transformed_one_hot) - grads_all.append(nest.pack_sequence_as(tensors_to_compute_grads, - grads_flat)) - - grads_lists = zip(*grads_all) - - else: + + try: + grads_lists = self._gradient_fns[self._estimation_mode]( + tensors_to_compute_grads) + except KeyError: raise ValueError("Unrecognized value {} for estimation_mode.".format( self._estimation_mode)) for grads_list, fb in zip(grads_lists, fisher_blocks_list): - fb.instantiate_factors(grads_list, damping) + fb.instantiate_factors(grads_list, self.damping) cov_updates = [ factor.make_covariance_update_op(cov_ema_decay) for factor in self._layers.get_factors() ] - inv_updates = { - op.name: op - for factor in self._layers.get_factors() - for op in factor.make_inverse_update_ops() - } + inv_updates = {op.name: op for op in self._get_all_inverse_update_ops()} return control_flow_ops.group(*cov_updates), control_flow_ops.group( *inv_updates.values()), inv_updates + + def _get_all_inverse_update_ops(self): + for factor in self._layers.get_factors(): + for op in factor.make_inverse_update_ops(): + yield op + + def _get_grads_lists_gradients(self, tensors): + grads_flat = gradients_impl.gradients(self._layers.total_sampled_loss(), + nest.flatten(tensors)) + grads_all = nest.pack_sequence_as(tensors, grads_flat) + return tuple((grad,) for grad in grads_all) + + def _get_grads_lists_empirical(self, tensors): + grads_flat = gradients_impl.gradients(self._layers.total_loss(), + nest.flatten(tensors)) + grads_all = nest.pack_sequence_as(tensors, grads_flat) + return tuple((grad,) for grad in grads_all) + + def _get_transformed_random_signs(self): + transformed_random_signs = [] + for loss in self._layers.losses: + transformed_random_signs.append( + loss.multiply_fisher_factor( + utils.generate_random_signs(loss.fisher_factor_inner_shape))) + return transformed_random_signs + + def _get_grads_lists_curvature_prop(self, tensors): + loss_inputs = list(loss.inputs for loss in self._layers.losses) + transformed_random_signs = self._get_transformed_random_signs() + grads_flat = gradients_impl.gradients( + nest.flatten(loss_inputs), + nest.flatten(tensors), + grad_ys=nest.flatten(transformed_random_signs)) + grads_all = nest.pack_sequence_as(tensors, grads_flat) + return tuple((grad,) for grad in grads_all) + + def _get_grads_lists_exact(self, tensors): + # Loop over all coordinates of all losses. + grads_all = [] + for loss in self._layers.losses: + for index in np.ndindex(*loss.fisher_factor_inner_static_shape[1:]): + transformed_one_hot = loss.multiply_fisher_factor_replicated_one_hot( + index) + grads_flat = gradients_impl.gradients( + loss.inputs, nest.flatten(tensors), grad_ys=transformed_one_hot) + grads_all.append(nest.pack_sequence_as(tensors, grads_flat)) + return zip(*grads_all) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py index 3bae45b32402c3ea60f3a82b99580d90dc150f86..efffaaef8d56aed3a1cdbf2df1d8209d58b3502f 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py @@ -12,7 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""FisherBlock definitions.""" +"""FisherBlock definitions. + +This library contains classes for estimating blocks in a model's Fisher +Information matrix. Suppose one has a model that parameterizes a posterior +distribution over 'y' given 'x' with parameters 'params', p(y | x, params). Its +Fisher Information matrix is given by, + + F(params) = E[ v(x, y, params) v(x, y, params)^T ] + +where, + + v(x, y, params) = (d / d params) log p(y | x, params) + +and the expectation is taken with respect to the data's distribution for 'x' and +the model's posterior distribution for 'y', + + x ~ p(x) + y ~ p(y | x, params) + +""" from __future__ import absolute_import from __future__ import division @@ -34,6 +53,14 @@ from tensorflow.python.ops import math_ops NORMALIZE_DAMPING_POWER = 1.0 +def set_global_constants(normalize_damping_power=None): + """Sets various global constants used by the classes in this module.""" + global NORMALIZE_DAMPING_POWER + + if normalize_damping_power is not None: + NORMALIZE_DAMPING_POWER = normalize_damping_power + + @six.add_metaclass(abc.ABCMeta) class FisherBlock(object): """Abstract base class for objects modeling approximate Fisher matrix blocks. @@ -87,6 +114,14 @@ class FisherBlock(object): """ pass + @abc.abstractproperty + def num_registered_minibatches(self): + """Number of minibatches registered for this FisherBlock. + + Typically equal to the number of towers in a multi-tower setup. + """ + pass + class FullFB(FisherBlock): """FisherBlock using a full matrix estimate (no approximations). @@ -125,8 +160,9 @@ class FullFB(FisherBlock): def multiply(self, vector): vector_flat = utils.tensors_to_column(vector) - out_flat = (math_ops.matmul(self._factor.get_cov(), vector_flat) + - self._damping * vector_flat) + out_flat = ( + math_ops.matmul(self._factor.get_cov(), vector_flat) + + self._damping * vector_flat) return utils.column_to_tensors(vector, out_flat) def full_fisher_block(self): @@ -136,6 +172,10 @@ class FullFB(FisherBlock): def tensors_to_compute_grads(self): return self._params + @property + def num_registered_minibatches(self): + return 1 # Multiple minibatches not supported. + class NaiveDiagonalFB(FisherBlock): """FisherBlock using a diagonal matrix approximation. @@ -181,62 +221,139 @@ class NaiveDiagonalFB(FisherBlock): def tensors_to_compute_grads(self): return self._params + @property + def num_registered_minibatches(self): + return 1 # Multiple minibatches not supported. + class FullyConnectedDiagonalFB(FisherBlock): """FisherBlock for fully-connected (dense) layers using a diagonal approx. - Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator - that is computed using the well-known trick. - """ + Estimates the Fisher Information matrix's diagonal entries for a fully + connected layer. Unlike NaiveDiagonalFB this uses the low-variance "sum of + squares" estimator. - # TODO(jamesmartens): add units tests for this class + Let 'params' be a vector parameterizing a model and 'i' an arbitrary index + into it. We are interested in Fisher(params)[i, i]. This is, - def __init__(self, layer_collection, inputs, outputs, has_bias=False): + Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] + = E[ v(x, y, params)[i] ^ 2 ] + + Consider fully connected layer in this model with (unshared) weight matrix + 'w'. For an example 'x' that produces layer inputs 'a' and output + preactivations 's', + + v(x, y, w) = vec( a (d loss / d s)^T ) + + This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding + to the layer's parameters 'w'. + """ + + def __init__(self, layer_collection, has_bias=False): """Creates a FullyConnectedDiagonalFB block. Args: layer_collection: The collection of all layers in the K-FAC approximate Fisher information matrix to which this FisherBlock belongs. - inputs: The Tensor of input activations to this layer. - outputs: The Tensor of output pre-activations from this layer. has_bias: Whether the component Kronecker factors have an additive bias. (Default: False) """ - self._inputs = inputs - self._outputs = outputs + self._inputs = [] + self._outputs = [] self._has_bias = has_bias super(FullyConnectedDiagonalFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + self._damping = damping self._factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedDiagonalFactor, (self._inputs, grads_list, - self._has_bias)) + fisher_factors.FullyConnectedDiagonalFactor, + (inputs, grads_list, self._has_bias)) def multiply_inverse(self, vector): + """Approximate damped inverse Fisher-vector product. + + Args: + vector: Tensor or 2-tuple of Tensors. if self._has_bias, Tensor of shape + [input_size, output_size] corresponding to layer's weights. If not, a + 2-tuple of the former and a Tensor of shape [output_size] corresponding + to the layer's bias. + + Returns: + Tensor of the same shape, corresponding to the inverse Fisher-vector + product. + """ reshaped_vect = utils.layer_params_to_mat2d(vector) reshaped_out = reshaped_vect / (self._factor.get_cov() + self._damping) return utils.mat2d_to_layer_params(vector, reshaped_out) def multiply(self, vector): + """Approximate damped Fisher-vector product. + + Args: + vector: Tensor or 2-tuple of Tensors. if self._has_bias, Tensor of shape + [input_size, output_size] corresponding to layer's weights. If not, a + 2-tuple of the former and a Tensor of shape [output_size] corresponding + to the layer's bias. + + Returns: + Tensor of the same shape, corresponding to the Fisher-vector product. + """ reshaped_vect = utils.layer_params_to_mat2d(vector) reshaped_out = reshaped_vect * (self._factor.get_cov() + self._damping) return utils.mat2d_to_layer_params(vector, reshaped_out) def tensors_to_compute_grads(self): + """Tensors to compute derivative of loss with respect to.""" return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, input_size]. Inputs to the + matrix-multiply. + outputs: Tensor of shape [batch_size, output_size]. Layer preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + + @property + def num_registered_minibatches(self): + result = len(self._inputs) + assert result == len(self._outputs) + return result + class ConvDiagonalFB(FisherBlock): """FisherBlock for convolutional layers using a diagonal approx. - Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" estimator. + Estimates the Fisher Information matrix's diagonal entries for a convolutional + layer. Unlike NaiveDiagonalFB this uses the low-variance "sum of squares" + estimator. + + Let 'params' be a vector parameterizing a model and 'i' an arbitrary index + into it. We are interested in Fisher(params)[i, i]. This is, + + Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i] + = E[ v(x, y, params)[i] ^ 2 ] + + Consider a convoluational layer in this model with (unshared) filter matrix + 'w'. For an example image 'x' that produces layer inputs 'a' and output + preactivations 's', + + v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T ) + + where 'loc' is a single (x, y) location in an image. + + This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding + to the layer's parameters 'w'. """ - # TODO(jamesmartens): add units tests for this class - def __init__(self, layer_collection, params, inputs, outputs, strides, - padding): + def __init__(self, layer_collection, params, strides, padding): """Creates a ConvDiagonalFB block. Args: @@ -246,37 +363,39 @@ class ConvDiagonalFB(FisherBlock): kernel alone, a Tensor of shape [kernel_height, kernel_width, in_channels, out_channels]. If kernel and bias, a tuple of 2 elements containing the previous and a Tensor of shape [out_channels]. - inputs: A Tensor of shape [batch_size, height, width, in_channels]. - Input activations to this layer. - outputs: A Tensor of shape [batch_size, height, width, out_channels]. - Output pre-activations from this layer. strides: The stride size in this layer (1-D Tensor of length 4). - padding: The padding in this layer (1-D of Tensor length 4). + padding: The padding in this layer (e.g. "SAME"). """ - self._inputs = inputs - self._outputs = outputs - self._strides = strides + self._inputs = [] + self._outputs = [] + self._strides = tuple(strides) if isinstance(strides, list) else strides self._padding = padding self._has_bias = isinstance(params, (tuple, list)) fltr = params[0] if self._has_bias else params self._filter_shape = tuple(fltr.shape.as_list()) - input_shape = tuple(inputs.shape.as_list()) - self._num_locations = (input_shape[1] * input_shape[2] - // (strides[1] * strides[2])) - super(ConvDiagonalFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): + # Concatenate inputs, grads_list into single Tensors. + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + + # Infer number of locations upon which convolution is applied. + inputs_shape = tuple(inputs.shape.as_list()) + self._num_locations = ( + inputs_shape[1] * inputs_shape[2] // + (self._strides[1] * self._strides[2])) + if NORMALIZE_DAMPING_POWER: damping /= self._num_locations ** NORMALIZE_DAMPING_POWER self._damping = damping self._factor = self._layer_collection.make_or_get_factor( fisher_factors.ConvDiagonalFactor, - (self._inputs, grads_list, self._filter_shape, self._strides, - self._padding, self._has_bias)) + (inputs, grads_list, self._filter_shape, self._strides, self._padding, + self._has_bias)) def multiply_inverse(self, vector): reshaped_vect = utils.layer_params_to_mat2d(vector) @@ -291,6 +410,22 @@ class ConvDiagonalFB(FisherBlock): def tensors_to_compute_grads(self): return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, height, width, input_size]. Inputs to + the convolution. + outputs: Tensor of shape [batch_size, height, width, output_size]. Layer + preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + + @property + def num_registered_minibatches(self): + return len(self._inputs) + class KroneckerProductFB(FisherBlock): """A base class for FisherBlocks with separate input and output factors. @@ -319,6 +454,14 @@ class KroneckerProductFB(FisherBlock): @property def _renorm_coeff(self): + """Kronecker factor multiplier coefficient. + + If this FisherBlock is represented as 'FB = c * kron(left, right)', then + this is 'c'. + + Returns: + 0-D Tensor. + """ return 1.0 def multiply_inverse(self, vector): @@ -337,10 +480,12 @@ class KroneckerProductFB(FisherBlock): left_factor = self._input_factor.get_cov() right_factor = self._output_factor.get_cov() reshaped_vector = utils.layer_params_to_mat2d(vector) - reshaped_out = (math_ops.matmul(reshaped_vector, right_factor) + - self._output_damping * reshaped_vector) - reshaped_out = (math_ops.matmul(left_factor, reshaped_out) + - self._input_damping * reshaped_out) + reshaped_out = ( + math_ops.matmul(reshaped_vector, right_factor) + + self._output_damping * reshaped_vector) + reshaped_out = ( + math_ops.matmul(left_factor, reshaped_out) + + self._input_damping * reshaped_out) if self._renorm_coeff != 1.0: reshaped_out *= math_ops.cast( self._renorm_coeff, dtype=reshaped_out.dtype) @@ -367,43 +512,90 @@ class FullyConnectedKFACBasicFB(KroneckerProductFB): K-FAC paper (https://arxiv.org/abs/1503.05671) """ - def __init__(self, layer_collection, inputs, outputs, has_bias=False): + def __init__(self, layer_collection, has_bias=False): """Creates a FullyConnectedKFACBasicFB block. Args: layer_collection: The collection of all layers in the K-FAC approximate Fisher information matrix to which this FisherBlock belongs. - inputs: The Tensor of input activations to this layer. - outputs: The Tensor of output pre-activations from this layer. has_bias: Whether the component Kronecker factors have an additive bias. (Default: False) """ - self._inputs = inputs - self._outputs = outputs + self._inputs = [] + self._outputs = [] self._has_bias = has_bias super(FullyConnectedKFACBasicFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): - self._input_factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedKroneckerFactor, ((self._inputs,), - self._has_bias)) - self._output_factor = self._layer_collection.make_or_get_factor( - fisher_factors.FullyConnectedKroneckerFactor, (grads_list,)) + """Instantiate Kronecker Factors for this FisherBlock. + + Args: + grads_list: List of list of Tensors. grads_list[i][j] is the + gradient of the loss with respect to 'outputs' from source 'i' and + tower 'j'. Each Tensor has shape [tower_minibatch_size, output_size]. + damping: 0-D Tensor or float. 'damping' * identity is approximately added + to this FisherBlock's Fisher approximation. + """ + # TODO(b/68033310): Validate which of, + # (1) summing on a single device (as below), or + # (2) on each device in isolation and aggregating + # is faster. + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + + self._input_factor = self._layer_collection.make_or_get_factor( # + fisher_factors.FullyConnectedKroneckerFactor, # + ((inputs,), self._has_bias)) + self._output_factor = self._layer_collection.make_or_get_factor( # + fisher_factors.FullyConnectedKroneckerFactor, # + (grads_list,)) self._register_damped_input_and_output_inverses(damping) def tensors_to_compute_grads(self): return self._outputs + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, input_size]. Inputs to the + matrix-multiply. + outputs: Tensor of shape [batch_size, output_size]. Layer preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + + @property + def num_registered_minibatches(self): + return len(self._inputs) + class ConvKFCBasicFB(KroneckerProductFB): """FisherBlock for 2D convolutional layers using the basic KFC approx. - See https://arxiv.org/abs/1602.01407 for details. + Estimates the Fisher Information matrix's blog for a convolutional + layer. + + Consider a convoluational layer in this model with (unshared) filter matrix + 'w'. For a minibatch that produces inputs 'a' and output preactivations 's', + this FisherBlock estimates, + + F(w) = #locations * kronecker(E[flat(a) flat(a)^T], + E[flat(ds) flat(ds)^T]) + + where + + ds = (d / ds) log p(y | x, w) + #locations = number of (x, y) locations where 'w' is applied. + + where the expectation is taken over all examples and locations and flat() + concatenates an array's leading dimensions. + + See equation 23 in https://arxiv.org/abs/1602.01407 for details. """ - def __init__(self, layer_collection, params, inputs, outputs, strides, - padding): + def __init__(self, layer_collection, params, strides, padding): """Creates a ConvKFCBasicFB block. Args: @@ -413,38 +605,43 @@ class ConvKFCBasicFB(KroneckerProductFB): kernel alone, a Tensor of shape [kernel_height, kernel_width, in_channels, out_channels]. If kernel and bias, a tuple of 2 elements containing the previous and a Tensor of shape [out_channels]. - inputs: A Tensor of shape [batch_size, height, width, in_channels]. - Input activations to this layer. - outputs: A Tensor of shape [batch_size, height, width, out_channels]. - Output pre-activations from this layer. strides: The stride size in this layer (1-D Tensor of length 4). padding: The padding in this layer (1-D of Tensor length 4). """ - self._inputs = inputs - self._outputs = outputs - self._strides = strides + self._inputs = [] + self._outputs = [] + self._strides = tuple(strides) if isinstance(strides, list) else strides self._padding = padding self._has_bias = isinstance(params, (tuple, list)) fltr = params[0] if self._has_bias else params self._filter_shape = tuple(fltr.shape.as_list()) - input_shape = tuple(inputs.shape.as_list()) - self._num_locations = (input_shape[1] * input_shape[2] // - (strides[1] * strides[2])) - super(ConvKFCBasicFB, self).__init__(layer_collection) def instantiate_factors(self, grads_list, damping): + # TODO(b/68033310): Validate which of, + # (1) summing on a single device (as below), or + # (2) on each device in isolation and aggregating + # is faster. + inputs = _concat_along_batch_dim(self._inputs) + grads_list = tuple(_concat_along_batch_dim(grads) for grads in grads_list) + + # Infer number of locations upon which convolution is applied. + self._num_locations = _num_conv_locations(inputs.shape.as_list(), + self._strides) + self._input_factor = self._layer_collection.make_or_get_factor( fisher_factors.ConvInputKroneckerFactor, - (self._inputs, self._filter_shape, self._strides, self._padding, + (inputs, self._filter_shape, self._strides, self._padding, self._has_bias)) self._output_factor = self._layer_collection.make_or_get_factor( fisher_factors.ConvOutputKroneckerFactor, (grads_list,)) if NORMALIZE_DAMPING_POWER: damping /= self._num_locations**NORMALIZE_DAMPING_POWER + self._damping = damping + self._register_damped_input_and_output_inverses(damping) @property @@ -453,3 +650,51 @@ class ConvKFCBasicFB(KroneckerProductFB): def tensors_to_compute_grads(self): return self._outputs + + def register_additional_minibatch(self, inputs, outputs): + """Registers an additional minibatch to the FisherBlock. + + Args: + inputs: Tensor of shape [batch_size, height, width, input_size]. Inputs to + the convolution. + outputs: Tensor of shape [batch_size, height, width, output_size]. Layer + preactivations. + """ + self._inputs.append(inputs) + self._outputs.append(outputs) + + @property + def num_registered_minibatches(self): + return len(self._inputs) + + +def _concat_along_batch_dim(tensor_list): + """Concatenate tensors along batch (first) dimension. + + Args: + tensor_list: list of Tensors or list of tuples of Tensors. + + Returns: + Tensor or tuple of Tensors. + + Raises: + ValueError: If 'tensor_list' is empty. + + """ + if not tensor_list: + raise ValueError( + "Cannot concatenate Tensors if there are no Tensors to concatenate.") + + if isinstance(tensor_list[0], (tuple, list)): + # [(tensor1a, tensor1b), + # (tensor2a, tensor2b), ...] --> (tensor_a, tensor_b) + return tuple( + array_ops.concat(tensors, axis=0) for tensors in zip(*tensor_list)) + else: + # [tensor1, tensor2] --> tensor + return array_ops.concat(tensor_list, axis=0) + + +def _num_conv_locations(input_shape, strides): + """Returns the number of locations a Conv kernel is applied to.""" + return input_shape[1] * input_shape[2] // (strides[1] * strides[2]) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py index c6cc169b3784ca2e60cde6cd703f13ddeaaad985..59389f8d385c18f50914d690cfaa2825ef807ed3 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks_lib.py @@ -31,7 +31,8 @@ _allowed_symbols = [ 'KroneckerProductFB', 'FullyConnectedKFACBasicFB', 'ConvKFCBasicFB', - 'ConvDiagonalFB' + 'ConvDiagonalFB', + 'set_global_constants', ] remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py index eacd9f53b1b1471ae6f77a35cbfcbb33d5434e2c..4e36813369e69de1d6f13ddb00566bda912244f6 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py @@ -33,9 +33,6 @@ from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.training import moving_averages -# TODO(someone): come up with a better mechanism to set these constants -# externally. See b/67084987 - # Whether to initialize covariance estimators at a zero matrix (or the identity # matrix). INIT_COVARIANCES_AT_ZERO = False @@ -53,6 +50,25 @@ EIGENVALUE_DECOMPOSITION_THRESHOLD = 2 EIGENVALUE_CLIPPING_THRESHOLD = 0.0 +def set_global_constants(init_covariances_at_zero=None, zero_debias=None, + eigenvalue_decomposition_threshold=None, + eigenvalue_clipping_threshold=None): + """Sets various global constants used by the classes in this module.""" + global INIT_COVARIANCES_AT_ZERO + global ZERO_DEBIAS + global EIGENVALUE_DECOMPOSITION_THRESHOLD + global EIGENVALUE_CLIPPING_THRESHOLD + + if init_covariances_at_zero is not None: + INIT_COVARIANCES_AT_ZERO = init_covariances_at_zero + if zero_debias is not None: + ZERO_DEBIAS = zero_debias + if eigenvalue_decomposition_threshold is not None: + EIGENVALUE_DECOMPOSITION_THRESHOLD = eigenvalue_decomposition_threshold + if eigenvalue_clipping_threshold is not None: + EIGENVALUE_CLIPPING_THRESHOLD = eigenvalue_clipping_threshold + + def inverse_initializer(shape, dtype, partition_info=None): # pylint: disable=unused-argument return array_ops.diag(array_ops.ones(shape[0], dtype)) @@ -412,11 +428,28 @@ class NaiveDiagonalFactor(DiagonalFactor): class FullyConnectedDiagonalFactor(DiagonalFactor): - """FisherFactor for a diagonal approx of a fully-connected layer's Fisher.""" + r"""FisherFactor for a diagonal approx of a fully-connected layer's Fisher. + + Given in = [batch_size, input_size] and out_grad = [batch_size, output_size], + approximates the covariance as, + + Cov(in, out) = (1/batch_size) \sum_{i} outer(in[i], out_grad[i]) ** 2.0 + + where the square is taken element-wise. + """ # TODO(jamesmartens): add units tests for this class def __init__(self, inputs, outputs_grads, has_bias=False): + """Instantiate FullyConnectedDiagonalFactor. + + Args: + inputs: Tensor of shape [batch_size, input_size]. Inputs to fully + connected layer. + outputs_grads: List of Tensors of shape [batch_size, output_size]. + Gradient of loss with respect to layer's preactivations. + has_bias: bool. If True, append '1' to each input. + """ self._outputs_grads = outputs_grads self._batch_size = array_ops.shape(inputs)[0] self._orig_tensors_name = scope_string_from_params((inputs,) + @@ -540,6 +573,14 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor): """ def __init__(self, tensors, has_bias=False): + """Instantiate FullyConnectedKroneckerFactor. + + Args: + tensors: List of Tensors of shape [batch_size, n]. Represents either a + layer's inputs or its output's gradients. + has_bias: bool. If True, assume this factor is for the layer's inputs and + append '1' to each row. + """ # The tensor argument is either a tensor of input activations or a tensor of # output pre-activation gradients. self._has_bias = has_bias @@ -568,9 +609,28 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor): class ConvInputKroneckerFactor(InverseProvidingFactor): - """Kronecker factor for the input side of a convolutional layer.""" + r"""Kronecker factor for the input side of a convolutional layer. + + Estimates E[ a a^T ] where a is the inputs to a convolutional layer given + example x. Expectation is taken over all examples and locations. + + Equivalent to \Omega in https://arxiv.org/abs/1602.01407 for details. See + Section 3.1 Estimating the factors. + """ def __init__(self, inputs, filter_shape, strides, padding, has_bias=False): + """Initializes ConvInputKroneckerFactor. + + Args: + inputs: Tensor of shape [batch_size, height, width, in_channels]. Inputs + to layer. + filter_shape: 1-D Tensor of length 4. Contains [kernel_height, + kernel_width, in_channels, out_channels]. + strides: 1-D Tensor of length 4. Contains [batch_stride, height_stride, + width_stride, in_channel_stride]. + padding: str. Padding method for layer. "SAME" or "VALID". + has_bias: bool. If True, append 1 to in_channel. + """ self._filter_shape = filter_shape self._strides = strides self._padding = padding @@ -618,9 +678,23 @@ class ConvInputKroneckerFactor(InverseProvidingFactor): class ConvOutputKroneckerFactor(InverseProvidingFactor): - """Kronecker factor for the output side of a convolutional layer.""" + r"""Kronecker factor for the output side of a convolutional layer. + + Estimates E[ ds ds^T ] where s is the preactivations of a convolutional layer + given example x and ds = (d / d s) log(p(y|x, w)). Expectation is taken over + all examples and locations. + + Equivalent to \Gamma in https://arxiv.org/abs/1602.01407 for details. See + Section 3.1 Estimating the factors. + """ def __init__(self, outputs_grads): + """Initializes ConvOutputKroneckerFactor. + + Args: + outputs_grads: list of Tensors. Each Tensor is of shape + [batch_size, height, width, out_channels]. + """ self._out_channels = outputs_grads[0].shape.as_list()[3] self._outputs_grads = outputs_grads super(ConvOutputKroneckerFactor, self).__init__() diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py b/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py index 49a07b15986b946105d32a1950bcccabaa363cef..23ee93cd405bbf719939df89d525c812ee061f8b 100644 --- a/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py +++ b/tensorflow/contrib/kfac/python/ops/fisher_factors_lib.py @@ -40,6 +40,7 @@ _allowed_symbols = [ "ConvInputKroneckerFactor", "ConvOutputKroneckerFactor", "ConvDiagonalFactor", + "set_global_constants", ] remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py index 1b77f5d3ba9820167e406dff3d55ef7d46d7482c..1806f5d8651e0b922fc30aed58d19de7faa5b265 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py @@ -27,6 +27,8 @@ from __future__ import print_function from collections import defaultdict from collections import OrderedDict +import six + from tensorflow.contrib.kfac.python.ops import fisher_blocks as fb from tensorflow.contrib.kfac.python.ops import loss_functions as lf from tensorflow.contrib.kfac.python.ops import utils @@ -37,10 +39,15 @@ from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import nest +# Names for various approximations that can be requested for Fisher blocks. APPROX_KRONECKER_NAME = "kron" APPROX_DIAGONAL_NAME = "diagonal" APPROX_FULL_NAME = "full" +# Possible value for 'reuse' keyword argument. Sets 'reuse' to +# tf.get_variable_scope().reuse. +VARIABLE_SCOPE = "VARIABLE_SCOPE" + # TODO(jamesmartens): need to add find_canonical_output back into this somewhere @@ -55,6 +62,7 @@ class LayerParametersDict(OrderedDict): super(LayerParametersDict, self).__init__(*args, **kwargs) def __setitem__(self, key, value): + key = self._canonicalize_key(key) tensors = key if isinstance(key, (tuple, list)) else (key,) key_collisions = self._tensors.intersection(tensors) if key_collisions: @@ -63,12 +71,26 @@ class LayerParametersDict(OrderedDict): super(LayerParametersDict, self).__setitem__(key, value) def __delitem__(self, key): + key = self._canonicalize_key(key) self._tensors.remove(key) super(LayerParametersDict, self).__delitem__(key) + def __getitem__(self, key): + key = self._canonicalize_key(key) + return super(LayerParametersDict, self).__getitem__(key) + + def __contains__(self, key): + key = self._canonicalize_key(key) + return super(LayerParametersDict, self).__contains__(key) + + def _canonicalize_key(self, key): + if isinstance(key, (list, tuple)): + return tuple(key) + return key -# TODO(duckworthd): add capability for LayerCollection to be "finalized" -# and do this when it gets used by FisherEstimator / KfacOptimizer + +# TODO(b/68034464): add capability for LayerCollection to be "finalized" +# and do this when it gets used by FisherEstimator / KfacOptimizer. class LayerCollection(object): @@ -94,13 +116,16 @@ class LayerCollection(object): self.fisher_factors = OrderedDict() self._generic_registrations = set() self._graph = graph or ops.get_default_graph() - self.losses = [] + self._loss_dict = {} # {str: LossFunction} self._subgraph = None with variable_scope.variable_scope(None, default_name=name) as scope: self._var_scope = scope.name - reset_internals = __init__ + @property + def losses(self): + """LossFunctions registered with this LayerCollection.""" + return list(self._loss_dict.values()) def register_block(self, layer_key, fisher_block): """Validates and registers the layer_key associated with the fisher_block. @@ -193,10 +218,10 @@ class LayerCollection(object): def get_use_count_map(self): """Returns a dict of variables to their number of registrations.""" vars_to_uses = defaultdict(int) - for key in self.fisher_blocks.keys(): + for key, block in six.iteritems(self.fisher_blocks): key = key if isinstance(key, (tuple, list)) else (key,) for k in key: - vars_to_uses[k] += 1 + vars_to_uses[k] += block.num_registered_minibatches return vars_to_uses def get_blocks(self): @@ -234,30 +259,118 @@ class LayerCollection(object): params, inputs, outputs, - approx=APPROX_KRONECKER_NAME): + approx=APPROX_KRONECKER_NAME, + reuse=VARIABLE_SCOPE): + """Registers a fully connnected layer. + + Args: + params: Tensor or 2-tuple of Tensors corresponding to weight and bias of + this layer. Weight matrix should have shape [input_size, output_size]. + Bias should have shape [output_size]. + inputs: Tensor of shape [batch_size, input_size]. Inputs to layer. + outputs: Tensor of shape [batch_size, output_size]. Preactivations + produced by layer. + approx: str. One of APPROX_KRONECKER_NAME or APPROX_DIAGONAL_NAME. + reuse: bool or str. If True, reuse an existing FisherBlock. If False, + create a new FisherBlock. If VARIABLE_SCOPE, use + tf.get_variable_scope().reuse. + + Raises: + ValueError: For improper value to 'approx'. + KeyError: If reuse == True but no FisherBlock found for 'params'. + ValueError: If reuse == True and FisherBlock found but of the wrong type. + """ + approx_to_block_types = { + APPROX_KRONECKER_NAME: fb.FullyConnectedKFACBasicFB, + APPROX_DIAGONAL_NAME: fb.FullyConnectedDiagonalFB, + } + + if approx not in approx_to_block_types: + raise ValueError("Bad value {} for approx.".format(approx)) + + block_type = approx_to_block_types[approx] has_bias = isinstance(params, (tuple, list)) - if approx == APPROX_KRONECKER_NAME: - self.register_block(params, - fb.FullyConnectedKFACBasicFB(self, inputs, outputs, - has_bias)) - elif approx == APPROX_DIAGONAL_NAME: - self.register_block(params, - fb.FullyConnectedDiagonalFB(self, inputs, outputs, - has_bias)) + + if reuse == VARIABLE_SCOPE: + reuse = variable_scope.get_variable_scope().reuse + + if reuse: + block = self.fisher_blocks.get(params, None) + if block is None: + raise KeyError( + "Reuse requested but no FisherBlock found for params {}.".format( + params)) + if not isinstance(block, block_type): + raise ValueError( + "Requested block of type {} but block of type {} already exists " + "for params {}.".format(block_type, type(block), params)) + else: + block = block_type(self, has_bias) + self.register_block(params, block) + + block.register_additional_minibatch(inputs, outputs) + + def register_conv2d(self, + params, + strides, + padding, + inputs, + outputs, + approx=APPROX_KRONECKER_NAME, + reuse=VARIABLE_SCOPE): + """Registers a convolutional layer. + + Args: + params: Tensor or 2-tuple of Tensors corresponding to weight and bias of + this layer. Weight matrix should have shape [kernel_height, + kernel_width, in_channels, out_channels]. Bias should have shape + [out_channels]. + strides: 1-D Tensor of length 4. Strides for convolution kernel. + padding: string. see tf.nn.conv2d for valid values. + inputs: Tensor of shape [batch_size, height, width, in_channels]. Inputs + to layer. + outputs: Tensor of shape [batch_size, height, width, out_channels]. + Preactivations produced by layer. + approx: str. One of APPROX_KRONECKER_NAME or APPROX_DIAGONAL_NAME. + reuse: bool or str. If True, reuse an existing FisherBlock. If False, + create a new FisherBlock. If VARIABLE_SCOPE, use + tf.get_variable_scope().reuse. + + Raises: + ValueError: For improper value to 'approx'. + KeyError: If reuse == True but no FisherBlock found for 'params'. + ValueError: If reuse == True and FisherBlock found but of the wrong type. + """ + approx_to_block_types = { + APPROX_KRONECKER_NAME: fb.ConvKFCBasicFB, + APPROX_DIAGONAL_NAME: fb.ConvDiagonalFB, + } + + if approx not in approx_to_block_types: raise ValueError("Bad value {} for approx.".format(approx)) - def register_conv2d(self, params, strides, padding, inputs, outputs, - approx=APPROX_KRONECKER_NAME): + block_type = approx_to_block_types[approx] - if approx == APPROX_KRONECKER_NAME: - self.register_block(params, - fb.ConvKFCBasicFB(self, params, inputs, outputs, - strides, padding)) - elif approx == APPROX_DIAGONAL_NAME: - self.register_block(params, - fb.ConvDiagonalFB(self, params, inputs, outputs, - strides, padding)) + if reuse == VARIABLE_SCOPE: + reuse = variable_scope.get_variable_scope().reuse + + if reuse: + block = self.fisher_blocks.get(params, None) + if block is None: + raise KeyError( + "Reuse requested but no FisherBlock found for params {}.".format( + params)) + if not isinstance(block, block_type): + raise ValueError( + "Requested block of type {} but block of type {} already exists " + "for params {}.".format(block_type, type(block), params)) + + else: + block = block_type(self, params, strides, padding) + self.register_block(params, block) + + block.register_additional_minibatch(inputs, outputs) def register_generic(self, params, batch_size, approx=APPROX_DIAGONAL_NAME): params = params if isinstance(params, (tuple, list)) else (params,) @@ -277,7 +390,9 @@ class LayerCollection(object): def register_categorical_predictive_distribution(self, logits, seed=None, - targets=None): + targets=None, + name=None, + reuse=VARIABLE_SCOPE): """Registers a categorical predictive distribution. Args: @@ -288,16 +403,55 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) + reuse: (OPTIONAL) bool or str. If True, reuse an existing FisherBlock. + If False, create a new FisherBlock. If VARIABLE_SCOPE, use + tf.get_variable_scope().reuse. + + Raises: + ValueError: If reuse=True and name != None. + ValueError: If reuse=True and seed != None. + KeyError: If reuse=True and no existing LossFunction with 'name' found. + KeyError: If reuse=False and existing LossFunction with 'name' found. """ - loss = lf.CategoricalLogitsNegativeLogProbLoss( - logits, targets=targets, seed=seed) - self.losses.append(loss) + name = name or self._graph.unique_name( + "register_categorical_predictive_distribution") + + if reuse == VARIABLE_SCOPE: + reuse = variable_scope.get_variable_scope().reuse + + if reuse: + if name is None: + raise ValueError( + "If reuse is enabled, loss function's name must be set.") + if seed is not None: + raise ValueError( + "Seed can only be specified at LossFunction instantiation.") + + loss = self._loss_dict.get(name, None) + + if loss is None: + raise KeyError( + "Unable to find loss function named {}. Create a new LossFunction " + "with reuse=False.".format(name)) + + loss.register_additional_minibatch(logits, targets=targets) + else: + if name in self._loss_dict: + raise KeyError( + "Loss function named {} already exists. Set reuse=True to append " + "another minibatch.".format(name)) + loss = lf.CategoricalLogitsNegativeLogProbLoss( + logits, targets=targets, seed=seed) + self._loss_dict[name] = loss def register_normal_predictive_distribution(self, mean, var=0.5, seed=None, - targets=None): + targets=None, + name=None): """Registers a normal predictive distribution. Args: @@ -312,15 +466,23 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_normal_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.NormalMeanNegativeLogProbLoss( mean, var, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def register_multi_bernoulli_predictive_distribution(self, logits, seed=None, - targets=None): + targets=None, + name=None): """Registers a multi-Bernoulli predictive distribution. Args: @@ -331,12 +493,40 @@ class LayerCollection(object): total_loss() is required, for example, to estimate the "empirical Fisher" (instead of the true Fisher). (Default: None) + name: (OPTIONAL) str or None. Unique name for this loss function. If None, + a new name is generated. (Default: None) """ + name = name or self._graph.unique_name( + "register_multi_bernoulli_predictive_distribution") + if name in self._loss_dict: + raise NotImplementedError( + "Adding logits to an existing LossFunction not yet supported.") loss = lf.MultiBernoulliNegativeLogProbLoss( logits, targets=targets, seed=seed) - self.losses.append(loss) + self._loss_dict[name] = loss def make_or_get_factor(self, cls, args): + """Insert 'cls(args)' into 'self.fisher_factors' if not already present. + + Wraps constructor in 'tf.variable_scope()' to ensure variables constructed + in 'cls.__init__' are placed under this LayerCollection's scope. + + Args: + cls: Class that implements FisherFactor. + args: Tuple of arguments to pass into 'cls's constructor. Must be + hashable. + + Returns: + Instance of 'cls' found in self.fisher_factors. + """ + try: + hash(args) + except TypeError: + raise TypeError(( + "Unable to use (cls, args) = ({}, {}) as a key in " + "LayerCollection.fisher_factors. The pair cannot be hashed." + ).format(cls, args)) + with variable_scope.variable_scope(self._var_scope): return utils.setdefault(self.fisher_factors, (cls, args), lambda: cls(*args)) diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py b/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py index 63a9b173bc809a7f25b382a3639462c27b39c5f9..d6bf61a210203dd74d4e93b65005f660b1fab4ff 100644 --- a/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py +++ b/tensorflow/contrib/kfac/python/ops/layer_collection_lib.py @@ -35,6 +35,7 @@ _allowed_symbols = [ "APPROX_KRONECKER_NAME", "APPROX_DIAGONAL_NAME", "APPROX_FULL_NAME", + "VARIABLE_SCOPE", ] remove_undocumented(__name__, allowed_exception_list=_allowed_symbols) diff --git a/tensorflow/contrib/kfac/python/ops/loss_functions.py b/tensorflow/contrib/kfac/python/ops/loss_functions.py index 979a4fd1de8f612a440f41f5ba0275c12bb3fce0..3cfde7f9ababab73980e93ea1dd65be1b559712b 100644 --- a/tensorflow/contrib/kfac/python/ops/loss_functions.py +++ b/tensorflow/contrib/kfac/python/ops/loss_functions.py @@ -42,8 +42,14 @@ class LossFunction(object): use this class. It depends on the use case. """ - def __init__(self, targets=None): - self._targets = targets + @abc.abstractproperty + def targets(self): + """The targets being predicted by the model. + + Returns: + None or Tensor of appropriate shape for calling self._evaluate() on. + """ + pass @abc.abstractproperty def inputs(self): @@ -51,16 +57,25 @@ class LossFunction(object): pass def evaluate(self): - """Evaluate the loss function.""" - if self._targets is not None: + """Evaluate the loss function on the targets.""" + if self.targets is not None: # We treat the targets as "constant". It's only the inputs that get # "back-propped" through. - return self._evaluate(array_ops.stop_gradient(self._targets)) + return self._evaluate(array_ops.stop_gradient(self.targets)) else: raise Exception("Cannot evaluate losses with unspecified targets.") @abc.abstractmethod def _evaluate(self, targets): + """Evaluates the log probability of the targets. + + Args: + targets: Tensor that distribution can calculate log_prob() of. + + Returns: + log probability of each target, summed across all targets. + """ + pass @abc.abstractmethod @@ -166,9 +181,9 @@ class LossFunction(object): class NegativeLogProbLoss(LossFunction): """Abstract base class for loss functions that are negative log probs.""" - def __init__(self, targets=None, seed=None): + def __init__(self, seed=None): self._default_seed = seed - super(NegativeLogProbLoss, self).__init__(targets=targets) + super(NegativeLogProbLoss, self).__init__() @property def inputs(self): @@ -176,6 +191,7 @@ class NegativeLogProbLoss(LossFunction): @abc.abstractproperty def params(self): + """Parameters to the underlying distribution.""" pass @abc.abstractmethod @@ -281,9 +297,18 @@ class NegativeLogProbLoss(LossFunction): @abc.abstractmethod def sample(self, seed): + """Sample 'targets' from the underlying distribution.""" pass def evaluate_on_sample(self, seed=None): + """Evaluates the log probability on a random sample. + + Args: + seed: int or None. Random seed for this draw from the distribution. + + Returns: + Log probability of sampled targets, summed across examples. + """ if seed is None: seed = self._default_seed # We treat the targets as "constant". It's only the inputs that get @@ -328,16 +353,19 @@ class NaturalParamsNegativeLogProbLoss(NegativeLogProbLoss): class DistributionNegativeLogProbLoss(NegativeLogProbLoss): """Base class for neg log prob losses that use the TF Distribution classes.""" - def __init__(self, dist, targets=None, seed=None): - self._dist = dist - super(DistributionNegativeLogProbLoss, self).__init__( - targets=targets, seed=seed) + def __init__(self, seed=None): + super(DistributionNegativeLogProbLoss, self).__init__(seed=seed) + + @abc.abstractproperty + def dist(self): + """The underlying tf.distributions.Distribution.""" + pass def _evaluate(self, targets): - return -math_ops.reduce_sum(self._dist.log_prob(targets)) + return -math_ops.reduce_sum(self.dist.log_prob(targets)) def sample(self, seed): - return self._dist.sample(seed=seed) + return self.dist.sample(seed=seed) class NormalMeanNegativeLogProbLoss(DistributionNegativeLogProbLoss, @@ -355,11 +383,18 @@ class NormalMeanNegativeLogProbLoss(DistributionNegativeLogProbLoss, """ def __init__(self, mean, var=0.5, targets=None, seed=None): - dist = normal.Normal(loc=mean, scale=var**0.5) self._mean = mean self._var = var - super(NormalMeanNegativeLogProbLoss, self).__init__( - dist, targets=targets, seed=seed) + self._targets = targets + super(NormalMeanNegativeLogProbLoss, self).__init__(seed=seed) + + @property + def targets(self): + return self._targets + + @property + def dist(self): + return normal.Normal(loc=self._mean, scale=math_ops.sqrt(self._var)) @property def params(self): @@ -416,10 +451,16 @@ class NormalMeanVarianceNegativeLogProbLoss(DistributionNegativeLogProbLoss): self._mean = mean self._variance = variance self._scale = math_ops.sqrt(variance) - dist = normal.Normal(loc=self._mean, scale=self._scale) - super(NormalMeanVarianceNegativeLogProbLoss, self).__init__(dist, - targets=targets, - seed=seed) + self._targets = targets + super(NormalMeanVarianceNegativeLogProbLoss, self).__init__(seed=seed) + + @property + def targets(self): + return self._targets + + @property + def dist(self): + return normal.Normal(loc=self._mean, scale=self._scale) @property def params(self): @@ -534,12 +575,53 @@ class CategoricalLogitsNegativeLogProbLoss(DistributionNegativeLogProbLoss, """ def __init__(self, logits, targets=None, seed=None): - dist = categorical.Categorical(logits=logits) - self._logits = logits - self._probs = dist.probs - self._sqrt_probs = math_ops.sqrt(self._probs) - super(CategoricalLogitsNegativeLogProbLoss, self).__init__( - dist, targets=targets, seed=seed) + """Instantiates a CategoricalLogitsNegativeLogProbLoss. + + Args: + logits: Tensor of shape [batch_size, output_size]. Parameters for + underlying distribution. + targets: None or Tensor of shape [output_size]. Each elements contains an + index in [0, output_size). + seed: int or None. Default random seed when sampling. + """ + self._logits_components = [] + self._targets_components = [] + self.register_additional_minibatch(logits, targets=targets) + super(CategoricalLogitsNegativeLogProbLoss, self).__init__(seed=seed) + + def register_additional_minibatch(self, logits, targets=None): + """Register an additiona minibatch's worth of parameters. + + Args: + logits: Tensor of shape [batch_size, output_size]. Parameters for + underlying distribution. + targets: None or Tensor of shape [batch_size, output_size]. Each row must + be a one-hot vector. + """ + self._logits_components.append(logits) + self._targets_components.append(targets) + + @property + def _logits(self): + return array_ops.concat(self._logits_components, axis=0) + + @property + def targets(self): + if all(target is None for target in self._targets_components): + return None + return array_ops.concat(self._targets_components, axis=0) + + @property + def dist(self): + return categorical.Categorical(logits=self._logits) + + @property + def _probs(self): + return self.dist.probs + + @property + def _sqrt_probs(self): + return math_ops.sqrt(self._probs) @property def params(self): @@ -595,12 +677,21 @@ class MultiBernoulliNegativeLogProbLoss(DistributionNegativeLogProbLoss, """ def __init__(self, logits, targets=None, seed=None): - dist = bernoulli.Bernoulli(logits=logits) self._logits = logits - self._probs = dist.probs + self._targets = targets + super(MultiBernoulliNegativeLogProbLoss, self).__init__(seed=seed) + + @property + def targets(self): + return self._targets - super(MultiBernoulliNegativeLogProbLoss, self).__init__( - dist, targets=targets, seed=seed) + @property + def dist(self): + return bernoulli.Bernoulli(logits=self._logits) + + @property + def _probs(self): + return self.dist.probs @property def params(self): @@ -632,11 +723,12 @@ class MultiBernoulliNegativeLogProbLoss(DistributionNegativeLogProbLoss, def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position): - """Inserts slice into a larger tensors of zeros. + """Inserts slice into a larger tensor of zeros. - Forms a new tensor that which is the same shape as slice_, except that + Forms a new tensor which is the same shape as slice_to_insert, except that the dimension given by 'dim' is expanded to the size given by 'dim_size'. - 'position' determines the position (index) of the slice in that dimension. + 'position' determines the position (index) at which to insert the slice within + that dimension. Assumes slice_to_insert.shape[dim] = 1. @@ -644,7 +736,7 @@ def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position): slice_to_insert: The slice to insert. dim: The dimension which to expand with zeros. dim_size: The new size of the 'dim' dimension. - position: The position of 'slice_' in the new tensor. + position: The position of 'slice_to_insert' in the new tensor. Returns: The new tensor. @@ -662,4 +754,4 @@ def insert_slice_in_zeros(slice_to_insert, dim, dim_size, position): before[dim] = position after[dim] = dim_size - position - 1 - return array_ops.pad(slice_to_insert, zip(before, after)) + return array_ops.pad(slice_to_insert, list(zip(before, after))) diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py index b34b4e10adb549990b63e9726a88294d03ecb59a..a7473481e44da0b09c047db9af29032918ea6cef 100644 --- a/tensorflow/contrib/kfac/python/ops/utils.py +++ b/tensorflow/contrib/kfac/python/ops/utils.py @@ -250,7 +250,7 @@ def generate_random_signs(shape, dtype=dtypes.float32): return 2 * math_ops.cast(ints, dtype=dtype) - 1 -def fwd_gradients(ys, xs, grad_xs=None): +def fwd_gradients(ys, xs, grad_xs=None, stop_gradients=None): """Compute forward-mode gradients.""" # See b/37888268. @@ -260,7 +260,8 @@ def fwd_gradients(ys, xs, grad_xs=None): # generated by the first gradients_impl.gradients call. us = [array_ops.zeros_like(y) + float("nan") for y in ys] - dydxs = gradients_impl.gradients(ys, xs, grad_ys=us) + dydxs = gradients_impl.gradients(ys, xs, grad_ys=us, + stop_gradients=stop_gradients) # Deal with strange types that gradients_impl.gradients returns but can't # deal with. diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD index 4eba29caecbddc408d168158daf8377aedab7bcc..894e6f6946bb59810a9da2d304cc0dd43d25201d 100644 --- a/tensorflow/contrib/labeled_tensor/BUILD +++ b/tensorflow/contrib/labeled_tensor/BUILD @@ -109,9 +109,9 @@ py_test( ":test_util", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", ], ) diff --git a/tensorflow/contrib/labeled_tensor/README.md b/tensorflow/contrib/labeled_tensor/README.md index 50c6750fd05f1dd605505011f74e23cb84eaf0b0..adce979e2acd1516d19572e31af1fd7b1c7a225c 100644 --- a/tensorflow/contrib/labeled_tensor/README.md +++ b/tensorflow/contrib/labeled_tensor/README.md @@ -3,6 +3,65 @@ LabeledTensor is a library for adding semantically meaningful dimension and coordinate labels to tensors in Tensorflow. -Maintainers: +LabeledTensor was inspired by [xarray](http://xarray.pydata.org) and +[pandas](http://pandas.pydata.org), projects that adds labels to NumPy array. + +## Data model + +`LabeledTensor` is an immutable object consisting of two components: + +- `tensor`: the `tf.Tensor` object containing the labeled tensor's data. +- `axes`: an OrderedDict-like object with keys given by axis names (e.g., + ``"channel"``) and values given by `Axis` objects. + +`Axis` objects keep track of the size of a dimension and, optionally, coordinate +labels along that axis (e.g., `("red", "green", "blue")`) in the form of a +tuple stored in `Axis.labels`. + +Operations on `LabeledTensors` use, preserve and transform axis names and +labels. + +## Quick start + +Try out the following snippet in a script or Jupyter notebook: + + import tensorflow as tf + + lt = tf.contrib.labeled_tensor + + # Create two LabeledTensors: + raw_image = tf.ones((299, 299, 3)) + axes = ['row', 'column', ('channel', ['red', 'green', 'blue'])] + image = lt.LabeledTensor(raw_image, axes) + assert image.tensor is raw_image + weights = lt.LabeledTensor(tf.constant([0.1, 0.3, 0.6]), + [image.axes['channel']]) + + # Examples of valid operations: + lt.transpose(image, ['column', 'row', 'channel']) + lt.reshape(image, ['row', 'column'], ['pixel']) + lt.concat([image, image], 'row') + lt.reduce_sum(image, ['channel']) + lt.select(image, {'channel': 'red'}) + lt.cast(image / 256.0, tf.uint8) + image * weights + lt.matmul(image[0, :, :], weights) + tf.cos(image) # automatically converts to tf.Tensor + +## Adding a custom op + +LabeledTensor has wrappers for [quite a +few](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/labeled_tensor/__init__.py) +TensorFlow ops. + +To easily add your own, you can use the `define_unary_op`, `define_binary_op` +and `define_reduce_op` functions, e.g., + + log = lt.define_unary_op('log', tf.log) + +## Questions + +Please reach out to the authors: + - Stephan Hoyer (shoyer@google.com, github.com/shoyer) - Eric Christiansen (ericmc@google.com, github.com/emchristiansen) diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD index bbb4fb1f57b54848e538d0cd1fad90ce0b6feab0..2f1f283811b6cb9e8bfb52ab2052afac1de700cb 100644 --- a/tensorflow/contrib/layers/BUILD +++ b/tensorflow/contrib/layers/BUILD @@ -88,17 +88,21 @@ tf_custom_op_py_library( "//tensorflow/python:clip_ops", "//tensorflow/python:common_shapes", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:data_flow_ops", "//tensorflow/python:embedding_ops", "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:init_ops", "//tensorflow/python:layers", + "//tensorflow/python:layers_base", + "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:random_ops", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:standard_ops", @@ -109,6 +113,7 @@ tf_custom_op_py_library( "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", "//tensorflow/python/feature_column", "@six_archive//:six", ], @@ -153,10 +158,10 @@ py_test( deps = [ ":layers_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", + "//tensorflow/python:session", "//third_party/py/numpy", ], ) @@ -168,9 +173,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":layers_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", "//third_party/py/numpy", @@ -238,6 +243,7 @@ py_test( ":layers_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:lookup_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:state_ops", @@ -280,9 +286,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":layers_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:variables", ], ) @@ -294,9 +300,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":layers_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//third_party/py/numpy", diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py index d8ab7c2d70d8a7346c04d326f3a51b40a4f900ea..d309ba958ded86afdc1e4bba2ff471a5181cda4e 100644 --- a/tensorflow/contrib/layers/__init__.py +++ b/tensorflow/contrib/layers/__init__.py @@ -47,6 +47,7 @@ See the @{$python/contrib.layers} guide. @@separable_conv2d @@separable_convolution2d @@softmax +@@spatial_softmax @@stack @@unit_norm @@bow_encoder diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py index 29ab281b1a603df153619eed2336420ddde9f6a8..c429d53cdc9101486359a09d985a5649c649f3e2 100644 --- a/tensorflow/contrib/layers/python/layers/layers.py +++ b/tensorflow/contrib/layers/python/layers/layers.py @@ -463,7 +463,8 @@ def batch_norm(inputs, scope=None, renorm=False, renorm_clipping=None, - renorm_decay=0.99): + renorm_decay=0.99, + adjustment=None): """Adds a Batch Normalization layer from http://arxiv.org/abs/1502.03167. "Batch Normalization: Accelerating Deep Network Training by Reducing @@ -546,6 +547,17 @@ def batch_norm(inputs, and should be neither too small (which would add noise) nor too large (which would give stale estimates). Note that `decay` is still applied to get the means and variances for inference. + adjustment: A function taking the `Tensor` containing the (dynamic) shape of + the input tensor and returning a pair (scale, bias) to apply to the + normalized values (before gamma and beta), only during training. For + example, + `adjustment = lambda shape: ( + tf.random_uniform(shape[-1:], 0.93, 1.07), + tf.random_uniform(shape[-1:], -0.1, 0.1))` + will scale the normalized value by up to 7% up or down, then shift the + result by up to 0.1 (with independent scaling and bias for each feature + but shared across all examples), and finally apply gamma and/or beta. If + `None`, no adjustment is applied. Returns: A `Tensor` representing the output of the operation. @@ -569,7 +581,10 @@ def batch_norm(inputs, # implementation in normalization_layers.BatchNormalization. inputs = ops.convert_to_tensor(inputs) rank = inputs.get_shape().ndims - possible_to_fuse = batch_weights is None and not renorm and rank in [2, 4] + possible_to_fuse = (batch_weights is None and + not renorm and + rank in [2, 4] and + adjustment is None) if fused and possible_to_fuse and ( zero_debias_moving_mean or rank == 2 or updates_collections is not ops.GraphKeys.UPDATE_OPS): @@ -636,6 +651,7 @@ def batch_norm(inputs, renorm=renorm, renorm_clipping=renorm_clipping, renorm_momentum=renorm_decay, + adjustment=adjustment, name=sc.name, _scope=sc, _reuse=reuse, @@ -1251,7 +1267,7 @@ def convolution2d_transpose( # Add variables to collections. _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias: + if layer.bias is not None: _add_variable_to_collections(layer.bias, variables_collections, 'biases') if normalizer_fn is not None: @@ -1360,7 +1376,7 @@ def convolution3d_transpose( # Add variables to collections. _add_variable_to_collections(layer.kernel, variables_collections, 'weights') - if layer.bias: + if layer.bias is not None: _add_variable_to_collections(layer.bias, variables_collections, 'biases') if normalizer_fn is not None: @@ -2506,7 +2522,7 @@ def separable_convolution2d( variables_collections, 'weights') _add_variable_to_collections(layer.pointwise_kernel, variables_collections, 'weights') - if layer.bias: + if layer.bias is not None: _add_variable_to_collections(layer.bias, variables_collections, 'biases') diff --git a/tensorflow/contrib/layers/python/layers/layers_test.py b/tensorflow/contrib/layers/python/layers/layers_test.py index 1040ad3ca7a4bbd56584f8e2cb8b2a2c8029d418..7c77e905f7432db4e42e7fda70aa72f32f40bb09 100644 --- a/tensorflow/contrib/layers/python/layers/layers_test.py +++ b/tensorflow/contrib/layers/python/layers/layers_test.py @@ -2644,6 +2644,26 @@ class BatchNormTest(test.TestCase): zero_debias_moving_mean=True) sess.run(variables_lib.global_variables_initializer()) + def testAdjustmentCreated(self): + # Tests that the adjustment is appropriately passed to and used by the core + # BN layer. + all_adjustments = [] + def _create_adjustment(shape): + adjustments = [array_ops.ones(shape[-1:]), array_ops.zeros(shape[-1:])] + all_adjustments.extend(adjustments) + return adjustments + depth = 8 + images = array_ops.zeros([10, 5, 5, depth]) + output = _layers.batch_norm( + images, + is_training=True, + adjustment=_create_adjustment) + self.assertListEqual(output.shape.as_list(), images.shape.as_list()) + self.assertEqual(len(all_adjustments), 2) + self.assertListEqual(all_adjustments[0].shape.as_list(), [depth]) + self.assertListEqual(all_adjustments[1].shape.as_list(), [depth]) + + class LayerNormTest(test.TestCase): def testUnknownShape(self): diff --git a/tensorflow/contrib/layers/python/layers/optimizers.py b/tensorflow/contrib/layers/python/layers/optimizers.py index 33db93b9704eb3c81d042e2636f916d5f685ad97..cdceea6fee5bdb5aeb6537ea55d25ccf107def4c 100644 --- a/tensorflow/contrib/layers/python/layers/optimizers.py +++ b/tensorflow/contrib/layers/python/layers/optimizers.py @@ -41,7 +41,7 @@ OPTIMIZER_CLS_NAMES = { "Adagrad": train.AdagradOptimizer, "Adam": train.AdamOptimizer, "Ftrl": train.FtrlOptimizer, - "Momentum": train.MomentumOptimizer, + "Momentum": lambda lr: train.MomentumOptimizer(lr, momentum=0.9), "RMSProp": train.RMSPropOptimizer, "SGD": train.GradientDescentOptimizer, } diff --git a/tensorflow/contrib/layers/python/layers/optimizers_test.py b/tensorflow/contrib/layers/python/layers/optimizers_test.py index 8813a99f1994ade17cca3b1371a17278e434cef9..1ea25bd1a5685eb6f840e621b5739029a660aa0f 100644 --- a/tensorflow/contrib/layers/python/layers/optimizers_test.py +++ b/tensorflow/contrib/layers/python/layers/optimizers_test.py @@ -176,7 +176,7 @@ class OptimizersTest(test.TestCase): session.run(train, feed_dict={x: 5}) var_value, global_step_value = session.run([var, global_step]) # Due to randomness the following number may change if graph is different. - self.assertAlmostEqual(var_value, 8.5591021, 4) + self.assertAlmostEqual(var_value, 9.86912, 4) self.assertEqual(global_step_value, 1) def testGradientNoiseWithClipping(self): @@ -193,7 +193,7 @@ class OptimizersTest(test.TestCase): variables.global_variables_initializer().run() session.run(train, feed_dict={x: 5}) var_value, global_step_value = session.run([var, global_step]) - self.assertAlmostEqual(var_value, 9.0, 4) + self.assertAlmostEqual(var_value, 9.86912, 4) self.assertEqual(global_step_value, 1) def testGradientClip(self): diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD index f3949beed04655456b3f0b550f5c757c85899270..ac615b120c16d5d9a7798874653f8f00f8fd15b4 100644 --- a/tensorflow/contrib/learn/BUILD +++ b/tensorflow/contrib/learn/BUILD @@ -768,7 +768,7 @@ py_test( ":learn", "//tensorflow/contrib/layers:layers_py", "//tensorflow/contrib/session_bundle:exporter", - "//tensorflow/contrib/session_bundle:manifest_proto_py", + "//tensorflow/contrib/session_bundle:manifest_proto_py_pb2", "//tensorflow/python:array_ops", "//tensorflow/python:client", "//tensorflow/python:client_testlib", diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py index 8bb1c83a451d7cd27f4df04f983cdd23d1e136ae..788d2d0b1a58fad16712c968593b40de0d3979f0 100644 --- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py +++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py @@ -981,9 +981,8 @@ class BaseEstimator( global_step = training_util.create_global_step(g) features, labels = input_fn() self._check_inputs(features, labels) - global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access - with ops.control_dependencies([global_step_read_tensor]): - model_fn_ops = self._get_train_ops(features, labels) + training_util._get_or_create_global_step_read() # pylint: disable=protected-access + model_fn_ops = self._get_train_ops(features, labels) ops.add_to_collection(ops.GraphKeys.LOSSES, model_fn_ops.loss) all_hooks.extend(hooks) all_hooks.extend([ diff --git a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py index bdb88b89bb3dba95a229724994874b0a26b1fc3f..4b34fc62849766370979bb2002d42ee03ea7161a 100644 --- a/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py +++ b/tensorflow/contrib/learn/python/learn/learn_io/graph_io.py @@ -442,7 +442,8 @@ def read_keyed_batch_features(file_pattern, feature_queue_capacity=100, num_enqueue_threads=2, parse_fn=None, - name=None): + name=None, + read_batch_size=None): """Adds operations to read, queue, batch and parse `Example` protos. Given file pattern (or list of files), will setup a queue for file names, @@ -482,6 +483,8 @@ def read_keyed_batch_features(file_pattern, parse_fn: Parsing function, takes `Example` Tensor returns parsed representation. If `None`, no parsing is done. name: Name of resulting op. + read_batch_size: An int or scalar `Tensor` specifying the number of + records to read at once. If `None`, defaults to `batch_size`. Returns: Returns tuple of: @@ -493,6 +496,7 @@ def read_keyed_batch_features(file_pattern, """ with ops.name_scope(name, 'read_batch_features', [file_pattern]) as scope: + if read_batch_size is None: read_batch_size = batch_size keys, examples = read_keyed_batch_examples( file_pattern, batch_size, @@ -501,7 +505,7 @@ def read_keyed_batch_features(file_pattern, num_epochs=num_epochs, queue_capacity=queue_capacity, num_threads=reader_num_threads, - read_batch_size=batch_size, + read_batch_size=read_batch_size, parse_fn=parse_fn, name=scope) # Parse the example. @@ -727,7 +731,8 @@ def read_batch_features(file_pattern, reader_num_threads=1, num_enqueue_threads=2, parse_fn=None, - name=None): + name=None, + read_batch_size=None): """Adds operations to read, queue, batch and parse `Example` protos. Given file pattern (or list of files), will setup a queue for file names, @@ -768,6 +773,8 @@ def read_batch_features(file_pattern, parse_fn: Parsing function, takes `Example` Tensor returns parsed representation. If `None`, no parsing is done. name: Name of resulting op. + read_batch_size: An int or scalar `Tensor` specifying the number of + records to read at once. If `None`, defaults to `batch_size`. Returns: A dict of `Tensor` or `SparseTensor` objects for each in `features`. @@ -786,6 +793,7 @@ def read_batch_features(file_pattern, reader_num_threads=reader_num_threads, feature_queue_capacity=feature_queue_capacity, num_enqueue_threads=num_enqueue_threads, + read_batch_size=read_batch_size, parse_fn=parse_fn, name=name) return features diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD index 734bac17dc82a61fd4c85b6277625d4a35961958..208e7bc69be76680868c766bc99429eea5870c80 100644 --- a/tensorflow/contrib/linalg/BUILD +++ b/tensorflow/contrib/linalg/BUILD @@ -17,22 +17,11 @@ py_library( srcs = ["__init__.py"] + glob(["python/ops/*.py"]), srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:common_shapes", - "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:linalg_ops", - "//tensorflow/python:math_ops", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//tensorflow/python/ops/linalg", - "//third_party/py/numpy", "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD index b8455477b0e39b54b6a5419ebd6ad41b2fc07912..b7b5418fe91e496f021b44fc32a33d2a549782e5 100644 --- a/tensorflow/contrib/lookup/BUILD +++ b/tensorflow/contrib/lookup/BUILD @@ -34,12 +34,12 @@ py_test( deps = [ ":lookup_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:framework_test_lib", "//tensorflow/python:lookup_ops", + "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", "//tensorflow/python:variables", diff --git a/tensorflow/contrib/losses/BUILD b/tensorflow/contrib/losses/BUILD index 33fbbe12d3926606c468d13bef2842b81a857edb..515290e2176169956f2bdcb881becc1170ac26e4 100644 --- a/tensorflow/contrib/losses/BUILD +++ b/tensorflow/contrib/losses/BUILD @@ -19,12 +19,19 @@ py_library( ], srcs_version = "PY2AND3", deps = [ + ":metric_learning_py", "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", + "//tensorflow/python:script_ops", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:summary", "//tensorflow/python:util", ], ) @@ -59,13 +66,16 @@ py_library( ], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:array_ops", + "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", - "//tensorflow/python:nn_ops", "//tensorflow/python:script_ops", + "//tensorflow/python:sparse_ops", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python:summary", "//tensorflow/python:util", ], ) @@ -78,18 +88,11 @@ py_test( srcs_version = "PY2AND3", deps = [ ":metric_learning_py", - "//tensorflow/contrib/framework:framework_py", - "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - "//tensorflow/python:random_seed", - "//tensorflow/python:training", - "//tensorflow/python:variable_scope", - "//tensorflow/python:variables", + "//tensorflow/python:nn", + "//tensorflow/python:sparse_tensor", "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/makefile/BUILD b/tensorflow/contrib/makefile/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..a8dd59f32a7f3b27993a7ee48ee7cc07ada59a4c --- /dev/null +++ b/tensorflow/contrib/makefile/BUILD @@ -0,0 +1,31 @@ +# Necessary build rules for makefile build in our CI. + +licenses(["notice"]) # Apache 2.0 + +package(default_visibility = ["//visibility:private"]) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) + +sh_test( + name = "build_all_linux", + size = "enormous", + srcs = ["build_all_linux.sh"], + data = [ + "//tensorflow:all_opensource_files", + "//third_party/eigen3:all_files", + "//third_party/fft2d:all_files", + ], + tags = [ + "manual", + "no_gpu", + "no_oss", + "notap", + ], +) diff --git a/tensorflow/contrib/makefile/Dockerfile b/tensorflow/contrib/makefile/Dockerfile index 341f22e692687fe24f4f4be596180ce0f8b16368..64d571a4edfffd82a82318b797ba1edf96f69027 100644 --- a/tensorflow/contrib/makefile/Dockerfile +++ b/tensorflow/contrib/makefile/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:16.04 -MAINTAINER Gunhan Gulsoy +LABEL maintainer="Gunhan Gulsoy " # Install make build dependencies for TensorFlow. RUN apt-get update diff --git a/tensorflow/contrib/makefile/Makefile b/tensorflow/contrib/makefile/Makefile index be7c790ee9e11ca90c0756011003a919f7d930f8..3b4d0ff799c05ce34cc55385ccc637467e443e40 100644 --- a/tensorflow/contrib/makefile/Makefile +++ b/tensorflow/contrib/makefile/Makefile @@ -14,7 +14,10 @@ # Host compilation settings # Find where we're running from, so we can store generated files here. -MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +ifeq ($(origin MAKEFILE_DIR), undefined) + MAKEFILE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +endif + HAS_GEN_HOST_PROTOC := \ $(shell test -f $(MAKEFILE_DIR)/gen/protobuf-host/bin/protoc && echo "true" ||\ echo "false") @@ -41,6 +44,11 @@ ifdef HEXAGON_LIBS endif endif # HEXAGON_LIBS +# If ANDROID_TYPES is not set assume __ANDROID_TYPES_SLIM__ +ifeq ($(ANDROID_TYPES),) + ANDROID_TYPES := -D__ANDROID_TYPES_SLIM__ +endif + # Try to figure out the host system HOST_OS := ifeq ($(OS),Windows_NT) @@ -71,6 +79,7 @@ HOST_LDOPTS += -L/usr/local/lib HOST_INCLUDES := \ -I. \ +-I$(MAKEFILE_DIR)/../../../ \ -I$(MAKEFILE_DIR)/downloads/ \ -I$(MAKEFILE_DIR)/downloads/eigen \ -I$(MAKEFILE_DIR)/downloads/gemmlowp \ @@ -190,6 +199,10 @@ LIBFLAGS := # If we're on OS X, make sure that globals aren't stripped out. ifeq ($(TARGET),OSX) +ifeq ($(HAS_GEN_HOST_PROTOC),true) + LIBFLAGS += -L$(MAKEFILE_DIR)/gen/protobuf-host/lib + export LD_LIBRARY_PATH=$(MAKEFILE_DIR)/gen/protobuf-host/lib +endif LDFLAGS += -all_load endif # Make sure that we don't strip global constructors on Linux. @@ -208,7 +221,7 @@ ifeq ($(TARGET),LINUX) endif # If we're cross-compiling for the Raspberry Pi, use the right gcc. ifeq ($(TARGET),PI) - CXXFLAGS += -D__ANDROID_TYPES_SLIM__ -DRASPBERRY_PI + CXXFLAGS += $(ANDROID_TYPES) -DRASPBERRY_PI LDFLAGS := -Wl,--no-whole-archive LIBS += -ldl -lpthread LIBFLAGS += -Wl,--allow-multiple-definition -Wl,--whole-archive @@ -330,7 +343,7 @@ ifeq ($(TARGET),IOS) -Wno-c++11-narrowing \ -mno-thumb \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONEOS_SYSROOT} @@ -354,7 +367,7 @@ ifeq ($(TARGET),IOS) -Wno-c++11-narrowing \ -mno-thumb \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONEOS_SYSROOT} @@ -377,7 +390,7 @@ ifeq ($(TARGET),IOS) -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONEOS_SYSROOT} @@ -401,7 +414,7 @@ ifeq ($(TARGET),IOS) -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONESIMULATOR_SYSROOT} @@ -424,7 +437,7 @@ ifeq ($(TARGET),IOS) -DUSE_GEMM_FOR_CONV \ -Wno-c++11-narrowing \ -DTF_LEAN_BINARY \ - -D__ANDROID_TYPES_SLIM__ \ + $(ANDROID_TYPES) \ -fno-exceptions \ -isysroot \ ${IPHONESIMULATOR_SYSROOT} @@ -502,6 +515,7 @@ $(wildcard tensorflow/core/platform/google/*) \ $(wildcard tensorflow/core/platform/google/*/*) \ $(wildcard tensorflow/core/platform/jpeg.*) \ $(wildcard tensorflow/core/platform/png.*) \ +$(wildcard tensorflow/core/platform/s3/*) \ $(wildcard tensorflow/core/platform/stream_executor.*) \ $(wildcard tensorflow/core/platform/windows/*) \ $(wildcard tensorflow/core/user_ops/*.cu.cc) \ diff --git a/tensorflow/contrib/makefile/build_all_linux.sh b/tensorflow/contrib/makefile/build_all_linux.sh index 5d73f697f4ef0b2a566deb04397b0def5a442cfa..a440633cfc23a7c606586a3b53180aaed6fe27ad 100755 --- a/tensorflow/contrib/makefile/build_all_linux.sh +++ b/tensorflow/contrib/makefile/build_all_linux.sh @@ -44,4 +44,5 @@ tensorflow/contrib/makefile/compile_linux_protobuf.sh # Build TensorFlow. make -j"${JOB_COUNT}" -f tensorflow/contrib/makefile/Makefile \ OPTFLAGS="-O3 -march=native" \ - HOST_CXXFLAGS="--std=c++11 -march=native" + HOST_CXXFLAGS="--std=c++11 -march=native" \ + MAKEFILE_DIR=$SCRIPT_DIR diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh index 39c89628d96ad1d7d8a28ec76071d4aa31085225..12e3f589306d54b10b38a48d8aed356de4ddc91b 100755 --- a/tensorflow/contrib/makefile/download_dependencies.sh +++ b/tensorflow/contrib/makefile/download_dependencies.sh @@ -20,11 +20,11 @@ DOWNLOADS_DIR=tensorflow/contrib/makefile/downloads BZL_FILE_PATH=tensorflow/workspace.bzl EIGEN_URL="$(grep -o 'http.*bitbucket.org/eigen/eigen/get/.*tar\.gz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" -GEMMLOWP_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" +GEMMLOWP_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/gemmlowp/.*zip' "${BZL_FILE_PATH}" | head -n1)" GOOGLETEST_URL="https://github.com/google/googletest/archive/release-1.8.0.tar.gz" -NSYNC_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" -PROTOBUF_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" -RE2_URL="$(grep -o 'http://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" +NSYNC_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/nsync/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" +PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" +RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)" FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)" # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64, @@ -54,7 +54,7 @@ download_and_extract() { elif [[ "${url}" == *zip ]]; then tempdir=$(mktemp -d) tempdir2=$(mktemp -d) - wget ${url} -P ${tempdir} + wget -P ${tempdir} ${url} unzip ${tempdir}/* -d ${tempdir2} # unzip has no strip components, so unzip to a temp dir, and move the files # we want from the tempdir to destination. diff --git a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt index 5ade8942af39f1d308c5f6e308e1cee754510926..938c4a53ab3fff72b028276eac5aad76ff01880d 100644 --- a/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt +++ b/tensorflow/contrib/makefile/proto_text_pb_cc_files.txt @@ -24,6 +24,7 @@ tensorflow/core/framework/summary.pb.cc tensorflow/core/framework/step_stats.pb.cc tensorflow/core/framework/resource_handle.pb.cc tensorflow/core/framework/remote_fused_graph_execute_info.pb.cc +tensorflow/core/framework/api_def.pb.cc tensorflow/core/framework/op_def.pb.cc tensorflow/core/framework/node_def.pb.cc tensorflow/core/framework/log_memory.pb.cc diff --git a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt index 1f0ad06cdc5b98ae9c08ea63dad70eb02b6ef46b..aa91b2f954504c42d33838c728abd666ef100e14 100644 --- a/tensorflow/contrib/makefile/proto_text_pb_h_files.txt +++ b/tensorflow/contrib/makefile/proto_text_pb_h_files.txt @@ -25,6 +25,7 @@ tensorflow/core/framework/summary.pb.h tensorflow/core/framework/step_stats.pb.h tensorflow/core/framework/resource_handle.pb.h tensorflow/core/framework/remote_fused_graph_execute_info.pb.h +tensorflow/core/framework/api_def.pb.h tensorflow/core/framework/op_def.pb.h tensorflow/core/framework/node_def.pb.h tensorflow/core/framework/log_memory.pb.h diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt index 1fda907074545d9b78a902182e4cec9e4212c22d..8b77c99cb574123c2af5d8f9f17cd403613cfffd 100644 --- a/tensorflow/contrib/makefile/tf_op_files.txt +++ b/tensorflow/contrib/makefile/tf_op_files.txt @@ -170,6 +170,8 @@ tensorflow/core/kernels/cwise_op_div.cc tensorflow/core/kernels/cwise_op_bitwise_xor.cc tensorflow/core/kernels/cwise_op_bitwise_or.cc tensorflow/core/kernels/cwise_op_bitwise_and.cc +tensorflow/core/kernels/cwise_op_left_shift.cc +tensorflow/core/kernels/cwise_op_right_shift.cc tensorflow/core/kernels/cwise_op_add_2.cc tensorflow/core/kernels/cwise_op_add_1.cc tensorflow/core/kernels/cwise_op_abs.cc @@ -262,3 +264,4 @@ tensorflow/core/kernels/spacetobatch_functor.cc tensorflow/core/kernels/spacetobatch_op.cc tensorflow/core/kernels/batchtospace_op.cc tensorflow/core/kernels/warn_about_ints.cc +tensorflow/core/kernels/segment_reduction_ops.cc diff --git a/tensorflow/contrib/makefile/tf_pb_text_files.txt b/tensorflow/contrib/makefile/tf_pb_text_files.txt index c39257ffa91fef184e8bd5258b19c4323a1b7fe0..b5431df2eb016d010c51bdbb33fd747b3569ce83 100644 --- a/tensorflow/contrib/makefile/tf_pb_text_files.txt +++ b/tensorflow/contrib/makefile/tf_pb_text_files.txt @@ -17,6 +17,7 @@ tensorflow/core/framework/summary.pb_text.cc tensorflow/core/framework/step_stats.pb_text.cc tensorflow/core/framework/resource_handle.pb_text.cc tensorflow/core/framework/remote_fused_graph_execute_info.pb_text.cc +tensorflow/core/framework/api_def.pb_text.cc tensorflow/core/framework/op_def.pb_text.cc tensorflow/core/framework/node_def.pb_text.cc tensorflow/core/framework/log_memory.pb_text.cc diff --git a/tensorflow/contrib/makefile/tf_proto_files.txt b/tensorflow/contrib/makefile/tf_proto_files.txt index a1a9aa7190205d9f3c34ef01b65db85f89f2ac85..d569bde637b20e0ca55c48c616855332abd9fb13 100644 --- a/tensorflow/contrib/makefile/tf_proto_files.txt +++ b/tensorflow/contrib/makefile/tf_proto_files.txt @@ -30,6 +30,7 @@ tensorflow/core/framework/step_stats.proto tensorflow/core/framework/resource_handle.proto tensorflow/core/framework/remote_fused_graph_execute_info.proto tensorflow/core/framework/reader_base.proto +tensorflow/core/framework/api_def.proto tensorflow/core/framework/op_def.proto tensorflow/core/framework/node_def.proto tensorflow/core/framework/log_memory.proto diff --git a/tensorflow/contrib/memory_stats/BUILD b/tensorflow/contrib/memory_stats/BUILD index 8b9d30dcfd088902ded36c7513ffc419e6bf7c7a..72424c32e7b756e6c50965f38135869e03ba730f 100644 --- a/tensorflow/contrib/memory_stats/BUILD +++ b/tensorflow/contrib/memory_stats/BUILD @@ -63,6 +63,8 @@ tf_custom_op_py_library( deps = [ ":memory_stats_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:platform", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/meta_graph_transform/BUILD b/tensorflow/contrib/meta_graph_transform/BUILD index d47ac5bcfe002ca8aaf4b8130c7b7fd58d1faeb9..4b5b1c3e15d36b7602791856416ece54d24798b2 100644 --- a/tensorflow/contrib/meta_graph_transform/BUILD +++ b/tensorflow/contrib/meta_graph_transform/BUILD @@ -21,7 +21,12 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/core:protos_all_py", - "//tensorflow/python:ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", + "//tensorflow/python:graph_util", + "//tensorflow/python:session", + "//tensorflow/python:training", + "//tensorflow/python:util", "//tensorflow/python/saved_model:constants", "//tensorflow/tools/graph_transforms:transform_graph_py", ], diff --git a/tensorflow/contrib/metrics/BUILD b/tensorflow/contrib/metrics/BUILD index e11dff08f853139fa19dd1dc418c4d3ac965ce71..9de664c822bf7a9abf7b8082f444c61dfa45f499 100644 --- a/tensorflow/contrib/metrics/BUILD +++ b/tensorflow/contrib/metrics/BUILD @@ -42,6 +42,7 @@ py_library( "//tensorflow/python:state_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python:weights_broadcast_ops", ], ) diff --git a/tensorflow/contrib/metrics/__init__.py b/tensorflow/contrib/metrics/__init__.py index 2c48882d0ea70bfdfa85730a2701c19cf76cb6e5..bb566f69029b4cd3b530c31bda22d78a19d9bf02 100644 --- a/tensorflow/contrib/metrics/__init__.py +++ b/tensorflow/contrib/metrics/__init__.py @@ -65,6 +65,7 @@ See the @{$python/contrib.metrics} guide. @@set_intersection @@set_size @@set_union +@@count """ from __future__ import absolute_import @@ -78,6 +79,7 @@ from tensorflow.contrib.metrics.python.ops.confusion_matrix_ops import confusion from tensorflow.contrib.metrics.python.ops.histogram_ops import auc_using_histogram from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metric_map from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metrics +from tensorflow.contrib.metrics.python.ops.metric_ops import count from tensorflow.contrib.metrics.python.ops.metric_ops import sparse_recall_at_top_k from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_accuracy from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py index 85c8e9038ac5642d0dbb20aea968474e0d7aa5f4..dbfc0934eacc6170a8521c1af54865ed0920c7c6 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py @@ -22,11 +22,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import collections as collections_lib + from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops -from tensorflow.python.ops import confusion_matrix from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics @@ -56,7 +57,10 @@ def _safe_div(numerator, denominator, name): name=name) -def _create_local(name, shape, collections=None, validate_shape=True, +def _create_local(name, + shape, + collections=None, + validate_shape=True, dtype=dtypes.float32): """Creates a new local variable. @@ -87,7 +91,9 @@ def _assert_weights_rank(weights, values): return check_ops.assert_rank_in(weights, (0, array_ops.rank(values))) -def _count_condition(values, weights=None, metrics_collections=None, +def _count_condition(values, + weights=None, + metrics_collections=None, updates_collections=None): """Sums the weights of cases where the given values are True. @@ -114,7 +120,7 @@ def _count_condition(values, weights=None, metrics_collections=None, or tuple. """ check_ops.assert_type(values, dtypes.bool) - count = _create_local('count', shape=[]) + count_ = _create_local('count', shape=[]) values = math_ops.to_float(values) if weights is not None: @@ -122,8 +128,8 @@ def _count_condition(values, weights=None, metrics_collections=None, with ops.control_dependencies((_assert_weights_rank(weights, values),)): values = math_ops.multiply(values, weights) - value_tensor = array_ops.identity(count) - update_op = state_ops.assign_add(count, math_ops.reduce_sum(values)) + value_tensor = array_ops.identity(count_) + update_op = state_ops.assign_add(count_, math_ops.reduce_sum(values)) if metrics_collections: ops.add_to_collections(metrics_collections, value_tensor) @@ -134,7 +140,9 @@ def _count_condition(values, weights=None, metrics_collections=None, return value_tensor, update_op -def streaming_true_positives(predictions, labels, weights=None, +def streaming_true_positives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -168,12 +176,17 @@ def streaming_true_positives(predictions, labels, weights=None, tuple. """ return metrics.true_positives( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_true_negatives(predictions, labels, weights=None, +def streaming_true_negatives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -206,20 +219,22 @@ def streaming_true_negatives(predictions, labels, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) - is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), - math_ops.equal(predictions, False)) + is_true_negative = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, False)) return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) -def streaming_false_positives(predictions, labels, weights=None, +def streaming_false_positives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -253,12 +268,17 @@ def streaming_false_positives(predictions, labels, weights=None, tuple. """ return metrics.false_positives( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_false_negatives(predictions, labels, weights=None, +def streaming_false_negatives(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -291,9 +311,12 @@ def streaming_false_negatives(predictions, labels, weights=None, or tuple. """ return metrics.false_negatives( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) # TODO(ptucker): Move this somewhere common, to share with ops/losses/losses.py. @@ -317,17 +340,18 @@ def _broadcast_weights(weights, values): with ops.name_scope(None, 'broadcast_weights', (values, weights)) as scope: weights_shape = weights.get_shape() values_shape = values.get_shape() - if (weights_shape.is_fully_defined() and - values_shape.is_fully_defined() and + if (weights_shape.is_fully_defined() and values_shape.is_fully_defined() and weights_shape.is_compatible_with(values_shape)): return weights with ops.control_dependencies((_assert_weights_rank(weights, values),)): - return math_ops.multiply( - weights, array_ops.ones_like(values), name=scope) + return math_ops.multiply(weights, array_ops.ones_like(values), name=scope) -def streaming_mean(values, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_mean(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the (weighted) mean of the given values. The `streaming_mean` function creates two local variables, `total` and `count` @@ -365,12 +389,18 @@ def streaming_mean(values, weights=None, metrics_collections=None, or tuple. """ return metrics.mean( - values=values, weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + values=values, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) -def streaming_mean_tensor(values, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_mean_tensor(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the element-wise (weighted) mean of the given tensors. In contrast to the `streaming_mean` function which returns a scalar with the @@ -412,12 +442,19 @@ def streaming_mean_tensor(values, weights=None, metrics_collections=None, or tuple. """ return metrics.mean_tensor( - values=values, weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) - + values=values, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) -def streaming_accuracy(predictions, labels, weights=None, - metrics_collections=None, updates_collections=None, +@deprecated(None, "Please switch to tf.metrics.accuracy. Note that the order " + "of the inputs of labels and predictions have been switched.") +def streaming_accuracy(predictions, + labels, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Calculates how often `predictions` matches `labels`. @@ -462,13 +499,19 @@ def streaming_accuracy(predictions, labels, weights=None, tuple. """ return metrics.accuracy( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_precision(predictions, labels, weights=None, - metrics_collections=None, updates_collections=None, +def streaming_precision(predictions, + labels, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Computes the precision of the predictions with respect to the labels. @@ -512,13 +555,19 @@ def streaming_precision(predictions, labels, weights=None, tuple. """ return metrics.precision( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_recall(predictions, labels, weights=None, - metrics_collections=None, updates_collections=None, +def streaming_recall(predictions, + labels, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Computes the recall of the predictions with respect to the labels. @@ -560,12 +609,17 @@ def streaming_recall(predictions, labels, weights=None, tuple. """ return metrics.recall( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def _true_negatives(labels, predictions, weights=None, +def _true_negatives(labels, + predictions, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -597,20 +651,22 @@ def _true_negatives(labels, predictions, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'true_negatives', (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'true_negatives', + (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) - is_true_negative = math_ops.logical_and(math_ops.equal(labels, False), - math_ops.equal(predictions, False)) + is_true_negative = math_ops.logical_and( + math_ops.equal(labels, False), math_ops.equal(predictions, False)) return _count_condition(is_true_negative, weights, metrics_collections, updates_collections) -def streaming_false_positive_rate(predictions, labels, weights=None, +def streaming_false_positive_rate(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -657,30 +713,35 @@ def streaming_false_positive_rate(predictions, labels, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_positive_rate', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + with variable_scope.variable_scope(name, 'false_positive_rate', + (predictions, labels, weights)): + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) false_p, false_positives_update_op = metrics.false_positives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) true_n, true_negatives_update_op = _true_negatives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_fpr(fp, tn, name): return array_ops.where( - math_ops.greater(fp + tn, 0), - math_ops.div(fp, fp + tn), - 0, - name) + math_ops.greater(fp + tn, 0), math_ops.div(fp, fp + tn), 0, name) fpr = compute_fpr(false_p, true_n, 'value') - update_op = compute_fpr( - false_positives_update_op, true_negatives_update_op, 'update_op') + update_op = compute_fpr(false_positives_update_op, true_negatives_update_op, + 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fpr) @@ -691,7 +752,9 @@ def streaming_false_positive_rate(predictions, labels, weights=None, return fpr, update_op -def streaming_false_negative_rate(predictions, labels, weights=None, +def streaming_false_negative_rate(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -738,30 +801,35 @@ def streaming_false_negative_rate(predictions, labels, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_negative_rate', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + with variable_scope.variable_scope(name, 'false_negative_rate', + (predictions, labels, weights)): + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions=math_ops.cast(predictions, dtype=dtypes.bool), labels=math_ops.cast(labels, dtype=dtypes.bool), weights=weights) false_n, false_negatives_update_op = metrics.false_negatives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) true_p, true_positives_update_op = metrics.true_positives( - labels, predictions, weights, metrics_collections=None, - updates_collections=None, name=None) + labels, + predictions, + weights, + metrics_collections=None, + updates_collections=None, + name=None) def compute_fnr(fn, tp, name): return array_ops.where( - math_ops.greater(fn + tp, 0), - math_ops.div(fn, fn + tp), - 0, - name) + math_ops.greater(fn + tp, 0), math_ops.div(fn, fn + tp), 0, name) fnr = compute_fnr(false_n, true_p, 'value') - update_op = compute_fnr( - false_negatives_update_op, true_positives_update_op, 'update_op') + update_op = compute_fnr(false_negatives_update_op, true_positives_update_op, + 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fnr) @@ -772,8 +840,11 @@ def streaming_false_negative_rate(predictions, labels, weights=None, return fnr, update_op -def _streaming_confusion_matrix_at_thresholds( - predictions, labels, thresholds, weights=None, includes=None): +def _streaming_confusion_matrix_at_thresholds(predictions, + labels, + thresholds, + weights=None, + includes=None): """Computes true_positives, false_negatives, true_negatives, false_positives. This function creates up to four local variables, `true_positives`, @@ -825,7 +896,7 @@ def _streaming_confusion_matrix_at_thresholds( if include not in all_includes: raise ValueError('Invaild key: %s.' % include) - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -861,8 +932,8 @@ def _streaming_confusion_matrix_at_thresholds( if weights is not None: broadcast_weights = weights_broadcast_ops.broadcast_weights( math_ops.to_float(weights), predictions) - weights_tiled = array_ops.tile(array_ops.reshape( - broadcast_weights, [1, -1]), [num_thresholds, 1]) + weights_tiled = array_ops.tile( + array_ops.reshape(broadcast_weights, [1, -1]), [num_thresholds, 1]) thresh_tiled.get_shape().assert_is_compatible_with( weights_tiled.get_shape()) else: @@ -877,8 +948,9 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_pos, pred_is_pos)) if weights_tiled is not None: is_true_positive *= weights_tiled - update_ops['tp'] = state_ops.assign_add( - true_positives, math_ops.reduce_sum(is_true_positive, 1)) + update_ops['tp'] = state_ops.assign_add(true_positives, + math_ops.reduce_sum( + is_true_positive, 1)) values['tp'] = true_positives if 'fn' in includes: @@ -887,8 +959,9 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_pos, pred_is_neg)) if weights_tiled is not None: is_false_negative *= weights_tiled - update_ops['fn'] = state_ops.assign_add( - false_negatives, math_ops.reduce_sum(is_false_negative, 1)) + update_ops['fn'] = state_ops.assign_add(false_negatives, + math_ops.reduce_sum( + is_false_negative, 1)) values['fn'] = false_negatives if 'tn' in includes: @@ -897,8 +970,9 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_neg, pred_is_neg)) if weights_tiled is not None: is_true_negative *= weights_tiled - update_ops['tn'] = state_ops.assign_add( - true_negatives, math_ops.reduce_sum(is_true_negative, 1)) + update_ops['tn'] = state_ops.assign_add(true_negatives, + math_ops.reduce_sum( + is_true_negative, 1)) values['tn'] = true_negatives if 'fp' in includes: @@ -907,36 +981,45 @@ def _streaming_confusion_matrix_at_thresholds( math_ops.logical_and(label_is_neg, pred_is_pos)) if weights_tiled is not None: is_false_positive *= weights_tiled - update_ops['fp'] = state_ops.assign_add( - false_positives, math_ops.reduce_sum(is_false_positive, 1)) + update_ops['fp'] = state_ops.assign_add(false_positives, + math_ops.reduce_sum( + is_false_positive, 1)) values['fp'] = false_positives return values, update_ops -def streaming_true_positives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_true_positives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('tp',)) return values['tp'], update_ops['tp'] -def streaming_false_negatives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_false_negatives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('fn',)) return values['fn'], update_ops['fn'] -def streaming_false_positives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_false_positives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('fp',)) return values['fp'], update_ops['fp'] -def streaming_true_negatives_at_thresholds( - predictions, labels, thresholds, weights=None): +def streaming_true_negatives_at_thresholds(predictions, + labels, + thresholds, + weights=None): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights=weights, includes=('tn',)) return values['tn'], update_ops['tn'] @@ -995,9 +1078,12 @@ def streaming_curve_points(labels=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + + TODO(chizeng): Consider rewriting this method to make use of logic within the + streaming_precision_recall_at_equal_thresholds method (to improve run time). """ - with variable_scope.variable_scope(name, 'curve_points', (labels, predictions, - weights)): + with variable_scope.variable_scope(name, 'curve_points', + (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': raise ValueError('curve must be either ROC or PR, %s unknown' % (curve)) kepsilon = 1e-7 # to account for floating point imprecisions @@ -1037,10 +1123,16 @@ def streaming_curve_points(labels=None, return points, update_op - -def streaming_auc(predictions, labels, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, - curve='ROC', name=None): +@deprecated(None, "Please switch to tf.metrics.auc. Note that the order of " + "the inputs of labels and predictions have been switched.") +def streaming_auc(predictions, + labels, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + curve='ROC', + name=None): """Computes the approximate AUC via a Riemann sum. The `streaming_auc` function creates four local variables, `true_positives`, @@ -1097,14 +1189,201 @@ def streaming_auc(predictions, labels, weights=None, num_thresholds=200, tuple. """ return metrics.auc( - predictions=predictions, labels=labels, weights=weights, - metrics_collections=metrics_collections, num_thresholds=num_thresholds, - curve=curve, updates_collections=updates_collections, name=name) + predictions=predictions, + labels=labels, + weights=weights, + metrics_collections=metrics_collections, + num_thresholds=num_thresholds, + curve=curve, + updates_collections=updates_collections, + name=name) + + +def streaming_precision_recall_at_equal_thresholds(predictions, + labels, + num_thresholds=None, + weights=None, + name=None, + use_locking=None): + """A helper method for creating metrics related to precision-recall curves. + + These values are true positives, false negatives, true negatives, false + positives, precision, and recall. This function returns a data structure that + contains ops within it. + + Unlike _streaming_confusion_matrix_at_thresholds (which exhibits O(T * N) + space and run time), this op exhibits O(T + N) space and run time, where T is + the number of thresholds and N is the size of the predictions tensor. Hence, + it may be advantageous to use this function when `predictions` is big. + + For instance, prefer this method for per-pixel classification tasks, for which + the predictions tensor may be very large. + + Each number in `predictions`, a float in `[0, 1]`, is compared with its + corresponding label in `labels`, and counts as a single tp/fp/tn/fn value at + each threshold. This is then multiplied with `weights` which can be used to + reweight certain values, or more commonly used for masking values. + + Args: + predictions: A floating point `Tensor` of arbitrary shape and whose values + are in the range `[0, 1]`. + labels: A bool `Tensor` whose shape matches `predictions`. + num_thresholds: Optional; Number of thresholds, evenly distributed in + `[0, 1]`. Should be `>= 2`. Defaults to 201. Note that the number of bins + is 1 less than `num_thresholds`. Using an even `num_thresholds` value + instead of an odd one may yield unfriendly edges for bins. + weights: Optional; If provided, a `Tensor` that has the same dtype as, + and broadcastable to, `predictions`. This tensor is multplied by counts. + name: Optional; variable_scope name. If not provided, the string + 'precision_recall_at_equal_threshold' is used. + use_locking: Optional; If True, the op will be protected by a lock. + Otherwise, the behavior is undefined, but may exhibit less contention. + Defaults to True. + + Returns: + result: A named tuple (See PrecisionRecallData within the implementation of + this function) with properties that are variables of shape + `[num_thresholds]`. The names of the properties are tp, fp, tn, fn, + precision, recall, thresholds. + update_op: An op that accumulates values. + + Raises: + ValueError: If `predictions` and `labels` have mismatched shapes, or if + `weights` is not `None` and its shape doesn't match `predictions`, or if + `includes` contains invalid keys. + """ + # Disable the invalid-name checker so that we can capitalize the name. + # pylint: disable=invalid-name + PrecisionRecallData = collections_lib.namedtuple( + 'PrecisionRecallData', + ['tp', 'fp', 'tn', 'fn', 'precision', 'recall', 'thresholds']) + # pylint: enable=invalid-name + + if num_thresholds is None: + num_thresholds = 201 + + if weights is None: + weights = 1.0 + + if use_locking is None: + use_locking = True + + check_ops.assert_type(labels, dtypes.bool) + + dtype = predictions.dtype + with variable_scope.variable_scope(name, + 'precision_recall_at_equal_thresholds', + (labels, predictions, weights)): + # Make sure that predictions are within [0.0, 1.0]. + with ops.control_dependencies([ + check_ops.assert_greater_equal( + predictions, + math_ops.cast(0.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]'), + check_ops.assert_less_equal( + predictions, + math_ops.cast(1.0, dtype=predictions.dtype), + message='predictions must be in [0, 1]') + ]): + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access + predictions=predictions, + labels=labels, + weights=weights) + predictions.get_shape().assert_is_compatible_with(labels.get_shape()) -def streaming_specificity_at_sensitivity( - predictions, labels, sensitivity, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, name=None): + # We cast to float to ensure we have 0.0 or 1.0. + f_labels = math_ops.cast(labels, dtype) + + # Get weighted true/false labels. + true_labels = f_labels * weights + false_labels = (1.0 - f_labels) * weights + + # Flatten predictions and labels. + predictions = array_ops.reshape(predictions, [-1]) + true_labels = array_ops.reshape(true_labels, [-1]) + false_labels = array_ops.reshape(false_labels, [-1]) + + # To compute TP/FP/TN/FN, we are measuring a binary classifier + # C(t) = (predictions >= t) + # at each threshold 't'. So we have + # TP(t) = sum( C(t) * true_labels ) + # FP(t) = sum( C(t) * false_labels ) + # + # But, computing C(t) requires computation for each t. To make it fast, + # observe that C(t) is a cumulative integral, and so if we have + # thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1} + # where n = num_thresholds, and if we can compute the bucket function + # B(i) = Sum( (predictions == t), t_i <= t < t{i+1} ) + # then we get + # C(t_i) = sum( B(j), j >= i ) + # which is the reversed cumulative sum in tf.cumsum(). + # + # We can compute B(i) efficiently by taking advantage of the fact that + # our thresholds are evenly distributed, in that + # width = 1.0 / (num_thresholds - 1) + # thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0] + # Given a prediction value p, we can map it to its bucket by + # bucket_index(p) = floor( p * (num_thresholds - 1) ) + # so we can use tf.scatter_add() to update the buckets in one pass. + # + # This implementation exhibits a run time and space complexity of O(T + N), + # where T is the number of thresholds and N is the size of predictions. + # Metrics that rely on _streaming_confusion_matrix_at_thresholds instead + # exhibit a complexity of O(T * N). + + # Compute the bucket indices for each prediction value. + bucket_indices = math_ops.cast( + math_ops.floor(predictions * (num_thresholds - 1)), dtypes.int32) + + with ops.name_scope('variables'): + tp_buckets_v = _create_local( + 'tp_buckets', shape=[num_thresholds], dtype=dtype) + fp_buckets_v = _create_local( + 'fp_buckets', shape=[num_thresholds], dtype=dtype) + + with ops.name_scope('update_op'): + update_tp = state_ops.scatter_add( + tp_buckets_v, bucket_indices, true_labels, use_locking=use_locking) + update_fp = state_ops.scatter_add( + fp_buckets_v, bucket_indices, false_labels, use_locking=use_locking) + + # Set up the cumulative sums to compute the actual metrics. + tp = math_ops.cumsum(tp_buckets_v, reverse=True, name='tp') + fp = math_ops.cumsum(fp_buckets_v, reverse=True, name='fp') + # fn = sum(true_labels) - tp + # = sum(tp_buckets) - tp + # = tp[0] - tp + # Similarly, + # tn = fp[0] - fp + tn = fp[0] - fp + fn = tp[0] - tp + + # We use a minimum to prevent division by 0. + epsilon = 1e-7 + precision = tp / math_ops.maximum(epsilon, tp + fp) + recall = tp / math_ops.maximum(epsilon, tp + fn) + + result = PrecisionRecallData( + tp=tp, + fp=fp, + tn=tn, + fn=fn, + precision=precision, + recall=recall, + thresholds=math_ops.lin_space(0.0, 1.0, num_thresholds)) + update_op = control_flow_ops.group(update_tp, update_fp) + return result, update_op + + +def streaming_specificity_at_sensitivity(predictions, + labels, + sensitivity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the specificity at a given sensitivity. The `streaming_specificity_at_sensitivity` function creates four local @@ -1154,15 +1433,24 @@ def streaming_specificity_at_sensitivity( or `updates_collections` are not a list or tuple. """ return metrics.specificity_at_sensitivity( - sensitivity=sensitivity, num_thresholds=num_thresholds, - predictions=predictions, labels=labels, weights=weights, + sensitivity=sensitivity, + num_thresholds=num_thresholds, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_sensitivity_at_specificity( - predictions, labels, specificity, weights=None, num_thresholds=200, - metrics_collections=None, updates_collections=None, name=None): +def streaming_sensitivity_at_specificity(predictions, + labels, + specificity, + weights=None, + num_thresholds=200, + metrics_collections=None, + updates_collections=None, + name=None): """Computes the sensitivity at a given specificity. The `streaming_sensitivity_at_specificity` function creates four local @@ -1212,16 +1500,25 @@ def streaming_sensitivity_at_specificity( or `updates_collections` are not a list or tuple. """ return metrics.sensitivity_at_specificity( - specificity=specificity, num_thresholds=num_thresholds, - predictions=predictions, labels=labels, weights=weights, + specificity=specificity, + num_thresholds=num_thresholds, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) - + updates_collections=updates_collections, + name=name) -def streaming_precision_at_thresholds(predictions, labels, thresholds, +@deprecated( + None, "Please switch to tf.metrics.precision_at_thresholds. Note that the " + "order of of the inputs of labels and predictions have been switched.") +def streaming_precision_at_thresholds(predictions, + labels, + thresholds, weights=None, metrics_collections=None, - updates_collections=None, name=None): + updates_collections=None, + name=None): """Computes precision values for different `thresholds` on `predictions`. The `streaming_precision_at_thresholds` function creates four local variables, @@ -1266,14 +1563,23 @@ def streaming_precision_at_thresholds(predictions, labels, thresholds, """ return metrics.precision_at_thresholds( thresholds=thresholds, - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) - + updates_collections=updates_collections, + name=name) -def streaming_recall_at_thresholds(predictions, labels, thresholds, - weights=None, metrics_collections=None, - updates_collections=None, name=None): +@deprecated( + None, "Please switch to tf.metrics.recall_at_thresholds. Note that the " + "order of of the inputs of labels and predictions have been switched.") +def streaming_recall_at_thresholds(predictions, + labels, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various recall values for different `thresholds` on `predictions`. The `streaming_recall_at_thresholds` function creates four local variables, @@ -1316,14 +1622,21 @@ def streaming_recall_at_thresholds(predictions, labels, thresholds, """ return metrics.recall_at_thresholds( thresholds=thresholds, - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_false_positive_rate_at_thresholds( - predictions, labels, thresholds, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_false_positive_rate_at_thresholds(predictions, + labels, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various fpr values for different `thresholds` on `predictions`. The `streaming_false_positive_rate_at_thresholds` function creates two @@ -1365,20 +1678,19 @@ def streaming_false_positive_rate_at_thresholds( either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_positive_rate_at_thresholds', - (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_positive_rate_at_thresholds', + (predictions, labels, weights)): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights, includes=('fp', 'tn')) # Avoid division by zero. epsilon = 1e-7 + def compute_fpr(fp, tn, name): return math_ops.div(fp, epsilon + fp + tn, name='fpr_' + name) fpr = compute_fpr(values['fp'], values['tn'], 'value') - update_op = compute_fpr( - update_ops['fp'], update_ops['tn'], 'update_op') + update_op = compute_fpr(update_ops['fp'], update_ops['tn'], 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fpr) @@ -1389,9 +1701,13 @@ def streaming_false_positive_rate_at_thresholds( return fpr, update_op -def streaming_false_negative_rate_at_thresholds( - predictions, labels, thresholds, weights=None, metrics_collections=None, - updates_collections=None, name=None): +def streaming_false_negative_rate_at_thresholds(predictions, + labels, + thresholds, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes various fnr values for different `thresholds` on `predictions`. The `streaming_false_negative_rate_at_thresholds` function creates two @@ -1433,20 +1749,19 @@ def streaming_false_negative_rate_at_thresholds( either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'false_negative_rate_at_thresholds', - (predictions, labels, weights)): + with variable_scope.variable_scope(name, 'false_negative_rate_at_thresholds', + (predictions, labels, weights)): values, update_ops = _streaming_confusion_matrix_at_thresholds( predictions, labels, thresholds, weights, includes=('fn', 'tp')) # Avoid division by zero. epsilon = 1e-7 + def compute_fnr(fn, tp, name): return math_ops.div(fn, epsilon + fn + tp, name='fnr_' + name) fnr = compute_fnr(values['fn'], values['tp'], 'value') - update_op = compute_fnr( - update_ops['fn'], update_ops['tp'], 'update_op') + update_op = compute_fnr(update_ops['fn'], update_ops['tp'], 'update_op') if metrics_collections: ops.add_to_collections(metrics_collections, fnr) @@ -1467,10 +1782,14 @@ def _at_k_name(name, k=None, class_id=None): return name -@deprecated('2016-11-08', 'Please use `streaming_sparse_recall_at_k`, ' - 'and reshape labels from [batch_size] to [batch_size, 1].') -def streaming_recall_at_k(predictions, labels, k, weights=None, - metrics_collections=None, updates_collections=None, +@deprecated("2016-11-08", "Please use `streaming_sparse_recall_at_k`, " + "and reshape labels from [batch_size] to [batch_size, 1].") +def streaming_recall_at_k(predictions, + labels, + k, + weights=None, + metrics_collections=None, + updates_collections=None, name=None): """Computes the recall@k of the predictions with respect to dense labels. @@ -1516,11 +1835,8 @@ def streaming_recall_at_k(predictions, labels, k, weights=None, tuple. """ in_top_k = math_ops.to_float(nn.in_top_k(predictions, labels, k)) - return streaming_mean(in_top_k, - weights, - metrics_collections, - updates_collections, - name or _at_k_name('recall', k)) + return streaming_mean(in_top_k, weights, metrics_collections, + updates_collections, name or _at_k_name('recall', k)) # TODO(ptucker): Validate range of values in labels? @@ -1599,10 +1915,14 @@ def streaming_sparse_recall_at_k(predictions, are not a list or tuple. """ return metrics.recall_at_k( - k=k, class_id=class_id, - predictions=predictions, labels=labels, weights=weights, + k=k, + class_id=class_id, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) # TODO(ptucker): Validate range of values in labels? @@ -1684,10 +2004,14 @@ def streaming_sparse_precision_at_k(predictions, are not a list or tuple. """ return metrics.sparse_precision_at_k( - k=k, class_id=class_id, - predictions=predictions, labels=labels, weights=weights, + k=k, + class_id=class_id, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) # TODO(ptucker): Validate range of values in labels? @@ -1766,10 +2090,9 @@ def streaming_sparse_precision_at_top_k(top_k_predictions, ValueError: If `top_k_predictions` has rank < 2. """ default_name = _at_k_name('precision', class_id=class_id) - with ops.name_scope( - name, default_name, - (top_k_predictions, labels, weights)) as name_scope: - return metrics_impl._sparse_precision_at_top_k( # pylint: disable=protected-access + with ops.name_scope(name, default_name, + (top_k_predictions, labels, weights)) as name_scope: + return metrics_impl.precision_at_top_k( labels=labels, predictions_idx=top_k_predictions, class_id=class_id, @@ -1848,8 +2171,8 @@ def sparse_recall_at_top_k(labels, are not a list or tuple. """ default_name = _at_k_name('recall', class_id=class_id) - with ops.name_scope(name, default_name, (top_k_predictions, labels, - weights)) as name_scope: + with ops.name_scope(name, default_name, + (top_k_predictions, labels, weights)) as name_scope: return metrics_impl._sparse_recall_at_top_k( # pylint: disable=protected-access labels=labels, predictions_idx=top_k_predictions, @@ -1919,9 +2242,13 @@ def streaming_sparse_average_precision_at_k(predictions, value matches `metric`. """ return metrics.sparse_average_precision_at_k( - k=k, predictions=predictions, labels=labels, weights=weights, + k=k, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) def streaming_sparse_average_precision_at_top_k(top_k_predictions, @@ -1986,8 +2313,10 @@ def streaming_sparse_average_precision_at_top_k(top_k_predictions, updates_collections=updates_collections, name=name) - -def streaming_mean_absolute_error(predictions, labels, weights=None, +@deprecated(None, "Please switch to tf.metrics.mean.") +def streaming_mean_absolute_error(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2035,12 +2364,18 @@ def streaming_mean_absolute_error(predictions, labels, weights=None, tuple. """ return metrics.mean_absolute_error( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_mean_relative_error(predictions, labels, normalizer, weights=None, +def streaming_mean_relative_error(predictions, + labels, + normalizer, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2089,12 +2424,18 @@ def streaming_mean_relative_error(predictions, labels, normalizer, weights=None, tuple. """ return metrics.mean_relative_error( - normalizer=normalizer, predictions=predictions, labels=labels, - weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + normalizer=normalizer, + predictions=predictions, + labels=labels, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) -def streaming_mean_squared_error(predictions, labels, weights=None, +def streaming_mean_squared_error(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2142,12 +2483,17 @@ def streaming_mean_squared_error(predictions, labels, weights=None, tuple. """ return metrics.mean_squared_error( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) -def streaming_root_mean_squared_error(predictions, labels, weights=None, +def streaming_root_mean_squared_error(predictions, + labels, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2195,9 +2541,12 @@ def streaming_root_mean_squared_error(predictions, labels, weights=None, tuple. """ return metrics.root_mean_squared_error( - predictions=predictions, labels=labels, weights=weights, + predictions=predictions, + labels=labels, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) def streaming_covariance(predictions, @@ -2253,12 +2602,12 @@ def streaming_covariance(predictions, ValueError: If labels and predictions are of different sizes or if either `metrics_collections` or `updates_collections` are not a list or tuple. """ - with variable_scope.variable_scope( - name, 'covariance', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + with variable_scope.variable_scope(name, 'covariance', + (predictions, labels, weights)): + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - count = _create_local('count', []) + count_ = _create_local('count', []) mean_prediction = _create_local('mean_prediction', []) mean_label = _create_local('mean_label', []) comoment = _create_local('comoment', []) # C_A in update equation @@ -2273,7 +2622,7 @@ def streaming_covariance(predictions, weighted_predictions = math_ops.multiply(predictions, weights) weighted_labels = math_ops.multiply(labels, weights) - update_count = state_ops.assign_add(count, batch_count) # n_AB in eqn + update_count = state_ops.assign_add(count_, batch_count) # n_AB in eqn prev_count = update_count - batch_count # n_A in update equation # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount) @@ -2298,34 +2647,34 @@ def streaming_covariance(predictions, # prev_mean_label is E[y_A] in the update equation prev_mean_label = update_mean_label - delta_mean_label - unweighted_batch_coresiduals = ( - (predictions - batch_mean_prediction) * (labels - batch_mean_label)) + unweighted_batch_coresiduals = ((predictions - batch_mean_prediction) * + (labels - batch_mean_label)) # batch_comoment is C_B in the update equation if weights is None: batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals) else: - batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals * - weights) + batch_comoment = math_ops.reduce_sum( + unweighted_batch_coresiduals * weights) # View delta_comoment as = C_AB - C_A in the update equation above. # Since C_A is stored in a var, by how much do we need to increment that var # to make the var = C_AB? - delta_comoment = (batch_comoment + - (prev_mean_prediction - batch_mean_prediction) * - (prev_mean_label - batch_mean_label) * - (prev_count * batch_count / update_count)) + delta_comoment = ( + batch_comoment + (prev_mean_prediction - batch_mean_prediction) * + (prev_mean_label - batch_mean_label) * + (prev_count * batch_count / update_count)) update_comoment = state_ops.assign_add(comoment, delta_comoment) covariance = array_ops.where( - math_ops.less_equal(count, 1.), + math_ops.less_equal(count_, 1.), float('nan'), - math_ops.truediv(comoment, count - 1), + math_ops.truediv(comoment, count_ - 1), name='covariance') with ops.control_dependencies([update_comoment]): update_op = array_ops.where( - math_ops.less_equal(count, 1.), + math_ops.less_equal(count_, 1.), float('nan'), - math_ops.truediv(comoment, count - 1), + math_ops.truediv(comoment, count_ - 1), name='update_op') if metrics_collections: @@ -2387,9 +2736,9 @@ def streaming_pearson_correlation(predictions, `weights` is the wrong size, or if either `metrics_collections` or `updates_collections` are not a `list` or `tuple`. """ - with variable_scope.variable_scope( - name, 'pearson_r', (predictions, labels, weights)): - predictions, labels, weights = _remove_squeezable_dimensions( + with variable_scope.variable_scope(name, 'pearson_r', + (predictions, labels, weights)): + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) # Broadcast weights here to avoid duplicate broadcasting in each call to @@ -2405,13 +2754,14 @@ def streaming_pearson_correlation(predictions, pearson_r = math_ops.truediv( cov, - math_ops.multiply(math_ops.sqrt(var_predictions), - math_ops.sqrt(var_labels)), + math_ops.multiply( + math_ops.sqrt(var_predictions), math_ops.sqrt(var_labels)), name='pearson_r') update_op = math_ops.truediv( update_cov, - math_ops.multiply(math_ops.sqrt(update_var_predictions), - math_ops.sqrt(update_var_labels)), + math_ops.multiply( + math_ops.sqrt(update_var_predictions), + math_ops.sqrt(update_var_labels)), name='update_op') if metrics_collections: @@ -2425,7 +2775,10 @@ def streaming_pearson_correlation(predictions, # TODO(nsilberman): add a 'normalized' flag so that the user can request # normalization if the inputs are not normalized. -def streaming_mean_cosine_distance(predictions, labels, dim, weights=None, +def streaming_mean_cosine_distance(predictions, + labels, + dim, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2467,16 +2820,15 @@ def streaming_mean_cosine_distance(predictions, labels, dim, weights=None, either `metrics_collections` or `updates_collections` are not a list or tuple. """ - predictions, labels, weights = _remove_squeezable_dimensions( + predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access predictions, labels, weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) radial_diffs = math_ops.multiply(predictions, labels) - radial_diffs = math_ops.reduce_sum(radial_diffs, - reduction_indices=[dim,], - keep_dims=True) - mean_distance, update_op = streaming_mean(radial_diffs, weights, - None, - None, + radial_diffs = math_ops.reduce_sum( + radial_diffs, reduction_indices=[ + dim, + ], keep_dims=True) + mean_distance, update_op = streaming_mean(radial_diffs, weights, None, None, name or 'mean_cosine_distance') mean_distance = math_ops.subtract(1.0, mean_distance) update_op = math_ops.subtract(1.0, update_op) @@ -2490,7 +2842,9 @@ def streaming_mean_cosine_distance(predictions, labels, dim, weights=None, return mean_distance, update_op -def streaming_percentage_less(values, threshold, weights=None, +def streaming_percentage_less(values, + threshold, + weights=None, metrics_collections=None, updates_collections=None, name=None): @@ -2530,9 +2884,12 @@ def streaming_percentage_less(values, threshold, weights=None, or tuple. """ return metrics.percentage_below( - values=values, threshold=threshold, weights=weights, + values=values, + threshold=threshold, + weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + updates_collections=updates_collections, + name=name) def streaming_mean_iou(predictions, @@ -2584,9 +2941,13 @@ def streaming_mean_iou(predictions, tuple. """ return metrics.mean_iou( - num_classes=num_classes, predictions=predictions, labels=labels, - weights=weights, metrics_collections=metrics_collections, - updates_collections=updates_collections, name=name) + num_classes=num_classes, + predictions=predictions, + labels=labels, + weights=weights, + metrics_collections=metrics_collections, + updates_collections=updates_collections, + name=name) def _next_array_size(required_size, growth_factor=1.5): @@ -2601,9 +2962,9 @@ def _next_array_size(required_size, growth_factor=1.5): tf.Tensor with dtype=int32 giving the next array size. """ exponent = math_ops.ceil( - math_ops.log(math_ops.cast(required_size, dtypes.float32)) - / math_ops.log(math_ops.cast(growth_factor, dtypes.float32))) - return math_ops.cast(math_ops.ceil(growth_factor ** exponent), dtypes.int32) + math_ops.log(math_ops.cast(required_size, dtypes.float32)) / math_ops.log( + math_ops.cast(growth_factor, dtypes.float32))) + return math_ops.cast(math_ops.ceil(growth_factor**exponent), dtypes.int32) def streaming_concat(values, @@ -2660,8 +3021,7 @@ def streaming_concat(values, if not 0 <= axis < ndim: raise ValueError('axis = %r not in [0, %r)' % (axis, ndim)) - fixed_shape = [dim.value for n, dim in enumerate(values_shape) - if n != axis] + fixed_shape = [dim.value for n, dim in enumerate(values_shape) if n != axis] if any(value is None for value in fixed_shape): raise ValueError('all dimensions of `values` other than the dimension to ' 'concatenate along must have statically known size') @@ -2770,54 +3130,71 @@ def aggregate_metric_map(names_to_tuples): return dict(zip(metric_names, value_ops)), dict(zip(metric_names, update_ops)) -def _remove_squeezable_dimensions(predictions, labels, weights): - """Squeeze last dim if needed. +def count(values, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): + """Computes the number of examples, or sum of `weights`. - Squeezes `predictions` and `labels` if their rank differs by 1. - Squeezes `weights` if its rank is 1 more than the new rank of `predictions` + When evaluating some metric (e.g. mean) on one or more subsets of the data, + this auxiliary metric is useful for keeping track of how many examples there + are in each subset. - This will use static shape if available. Otherwise, it will add graph - operations, which could result in a performance hit. + If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. Args: - predictions: Predicted values, a `Tensor` of arbitrary dimensions. - labels: Label values, a `Tensor` whose dimensions match `predictions`. - weights: Optional weight `Tensor`. It will be squeezed if its rank is 1 - more than the new rank of `predictions` + values: A `Tensor` of arbitrary dimensions. Only it's shape is used. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions + must be either `1`, or the same as the corresponding `labels` + dimension). + metrics_collections: An optional list of collections that the metric + value variable should be added to. + updates_collections: An optional list of collections that the metric update + ops should be added to. + name: An optional variable_scope name. Returns: - Tuple of `predictions`, `labels` and `weights`, possibly with the last - dimension squeezed. + count: A `Tensor` representing the current value of the metric. + update_op: An operation that accumulates the metric from a batch of data. + + Raises: + ValueError: If `weights` is not `None` and its shape doesn't match `values`, + or if either `metrics_collections` or `updates_collections` are not a list + or tuple. """ - labels, predictions = confusion_matrix.remove_squeezable_dimensions( - labels, predictions) - predictions.get_shape().assert_is_compatible_with(labels.get_shape()) - if weights is not None: - weights = ops.convert_to_tensor(weights) - predictions_shape = predictions.get_shape() - predictions_rank = predictions_shape.ndims - weights_shape = weights.get_shape() - weights_rank = weights_shape.ndims - - if (predictions_rank is not None) and (weights_rank is not None): - # Use static rank. - if weights_rank - predictions_rank == 1: - weights = array_ops.squeeze(weights, [-1]) - elif (weights_rank is None) or ( - weights_shape.dims[-1].is_compatible_with(1)): - # Use dynamic rank - weights = control_flow_ops.cond( - math_ops.equal(array_ops.rank(weights), - math_ops.add(array_ops.rank(predictions), 1)), - lambda: array_ops.squeeze(weights, [-1]), - lambda: weights) - return predictions, labels, weights + with variable_scope.variable_scope(name, 'count', (values, weights)): + count_ = _create_local('count', shape=[]) + + if weights is None: + num_values = math_ops.to_float(array_ops.size(values)) + else: + _, _, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access + predictions=values, + labels=None, + weights=weights) + weights = weights_broadcast_ops.broadcast_weights( + math_ops.to_float(weights), values) + num_values = math_ops.reduce_sum(weights) + + with ops.control_dependencies([values]): + update_op = state_ops.assign_add(count_, num_values) + + if metrics_collections: + ops.add_to_collections(metrics_collections, count_) + + if updates_collections: + ops.add_to_collections(updates_collections, update_op) + + return count_, update_op __all__ = [ 'aggregate_metric_map', 'aggregate_metrics', + 'count', 'sparse_recall_at_top_k', 'streaming_accuracy', 'streaming_auc', diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py index c5fcc20abd4927c5408071bae8fa8620cd4d7eb2..6a8284786f592b2fe840e3c68099fecc93dc91c6 100644 --- a/tensorflow/contrib/metrics/python/ops/metric_ops_test.py +++ b/tensorflow/contrib/metrics/python/ops/metric_ops_test.py @@ -1101,7 +1101,7 @@ class StreamingPrecisionTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) precision, update_op = metrics.streaming_precision(predictions, labels) with self.test_session() as sess: @@ -1265,7 +1265,7 @@ class StreamingRecallTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) recall, update_op = metrics.streaming_recall(predictions, labels) with self.test_session() as sess: @@ -1388,7 +1388,7 @@ class StreamingFPRTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) fpr, update_op = metrics.streaming_false_positive_rate( predictions, labels) @@ -1516,7 +1516,7 @@ class StreamingFNRTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) fnr, update_op = metrics.streaming_false_negative_rate( predictions, labels) @@ -1737,7 +1737,7 @@ class StreamingAUCTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) auc, update_op = metrics.streaming_auc(predictions, labels) with self.test_session() as sess: @@ -1970,6 +1970,170 @@ class StreamingAUCTest(test.TestCase): self.assertAlmostEqual(expected_auc, auc.eval(), 2) +class StreamingPrecisionRecallAtEqualThresholdsTest(test.TestCase): + + def setUp(self): + np.random.seed(1) + ops.reset_default_graph() + + def _testResultsEqual(self, expected_dict, gotten_result): + """Tests that 2 results (dicts) represent the same data. + + Args: + expected_dict: A dictionary with keys that are the names of properties + of PrecisionRecallData and whose values are lists of floats. + gotten_result: A PrecisionRecallData object. + """ + gotten_dict = {k: t.eval() for k, t in gotten_result._asdict().items()} + self.assertItemsEqual( + list(expected_dict.keys()), list(gotten_dict.keys())) + + for key, expected_values in expected_dict.items(): + self.assertAllClose(expected_values, gotten_dict[key]) + + def _testCase(self, predictions, labels, expected_result, weights=None): + """Performs a test given a certain scenario of labels, predictions, weights. + + Args: + predictions: The predictions tensor. Of type float32. + labels: The labels tensor. Of type bool. + expected_result: The expected result (dict) that maps to tensors. + weights: Optional weights tensor. + """ + with self.test_session() as sess: + predictions_tensor = constant_op.constant( + predictions, dtype=dtypes_lib.float32) + labels_tensor = constant_op.constant(labels, dtype=dtypes_lib.bool) + weights_tensor = None + if weights: + weights_tensor = constant_op.constant(weights, dtype=dtypes_lib.float32) + gotten_result, update_op = ( + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=predictions_tensor, + labels=labels_tensor, + num_thresholds=3, + weights=weights_tensor)) + + sess.run(variables.local_variables_initializer()) + sess.run(update_op) + + self._testResultsEqual(expected_result, gotten_result) + + def testVars(self): + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32), + labels=constant_op.constant([True], dtype=dtypes_lib.bool)) + _assert_local_variables( + self, + ( + 'precision_recall_at_equal_thresholds/variables/tp_buckets:0', + 'precision_recall_at_equal_thresholds/variables/fp_buckets:0' + )) + + def testVarsWithName(self): + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=constant_op.constant([0.42], dtype=dtypes_lib.float32), + labels=constant_op.constant([True], dtype=dtypes_lib.bool), + name='foo') + _assert_local_variables( + self, ('foo/variables/tp_buckets:0', 'foo/variables/fp_buckets:0')) + + def testValuesAreIdempotent(self): + predictions = constant_op.constant( + np.random.uniform(size=(10, 3)), dtype=dtypes_lib.float32) + labels = constant_op.constant( + np.random.uniform(size=(10, 3)) > 0.5, dtype=dtypes_lib.bool) + + result, update_op = ( + metric_ops.streaming_precision_recall_at_equal_thresholds( + predictions=predictions, labels=labels)) + + with self.test_session() as sess: + # Run several updates. + sess.run(variables.local_variables_initializer()) + for _ in range(3): + sess.run(update_op) + + # Then verify idempotency. + initial_result = {k: value.eval().tolist() for k, value in + result._asdict().items()} + for _ in range(3): + self._testResultsEqual(initial_result, result) + + def testAllTruePositives(self): + self._testCase([[1]], [[True]], { + 'tp': [1, 1, 1], + 'fp': [0, 0, 0], + 'tn': [0, 0, 0], + 'fn': [0, 0, 0], + 'precision': [1.0, 1.0, 1.0], + 'recall': [1.0, 1.0, 1.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testAllTrueNegatives(self): + self._testCase([[0]], [[False]], { + 'tp': [0, 0, 0], + 'fp': [1, 0, 0], + 'tn': [0, 1, 1], + 'fn': [0, 0, 0], + 'precision': [0.0, 0.0, 0.0], + 'recall': [0.0, 0.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testAllFalsePositives(self): + self._testCase([[1]], [[False]], { + 'tp': [0, 0, 0], + 'fp': [1, 1, 1], + 'tn': [0, 0, 0], + 'fn': [0, 0, 0], + 'precision': [0.0, 0.0, 0.0], + 'recall': [0.0, 0.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testAllFalseNegatives(self): + self._testCase([[0]], [[True]], { + 'tp': [1, 0, 0], + 'fp': [0, 0, 0], + 'tn': [0, 0, 0], + 'fn': [0, 1, 1], + 'precision': [1.0, 0.0, 0.0], + 'recall': [1.0, 0.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testManyValues(self): + self._testCase( + [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], + [[True, False, False, True, True, True]], + { + 'tp': [4, 3, 0], + 'fp': [2, 0, 0], + 'tn': [0, 2, 2], + 'fn': [0, 1, 4], + 'precision': [2.0 / 3.0, 1.0, 0.0], + 'recall': [1.0, 0.75, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }) + + def testManyValuesWithWeights(self): + self._testCase( + [[0.2, 0.3, 0.4, 0.6, 0.7, 0.8]], + [[True, False, False, True, True, True]], + { + 'tp': [1.5, 1.5, 0.0], + 'fp': [2.5, 0.0, 0.0], + 'tn': [0.0, 2.5, 2.5], + 'fn': [0.0, 0.0, 1.5], + 'precision': [0.375, 1.0, 0.0], + 'recall': [1.0, 1.0, 0.0], + 'thresholds': [0.0, 0.5, 1.0], + }, + weights=[[0.0, 0.5, 2.0, 0.0, 0.5, 1.0]]) + + class StreamingSpecificityAtSensitivityTest(test.TestCase): def setUp(self): @@ -2009,7 +2173,7 @@ class StreamingSpecificityAtSensitivityTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) specificity, update_op = metrics.streaming_specificity_at_sensitivity( predictions, labels, sensitivity=0.7) @@ -2271,7 +2435,7 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) thresholds = [0, 0.5, 1.0] prec, prec_op = metrics.streaming_precision_at_thresholds(predictions, labels, @@ -2282,12 +2446,14 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): with self.test_session() as sess: sess.run(variables.local_variables_initializer()) - # Run several updates, then verify idempotency. - sess.run([prec_op, rec_op]) + # Run several updates. + for _ in range(10): + sess.run([prec_op, rec_op]) + + # Then verify idempotency. initial_prec = prec.eval() initial_rec = rec.eval() for _ in range(10): - sess.run([prec_op, rec_op]) self.assertAllClose(initial_prec, prec.eval()) self.assertAllClose(initial_rec, rec.eval()) @@ -2361,14 +2527,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.streaming_recall_at_thresholds( predictions, labels, thresholds, weights=weights) - [prec_low, prec_high] = array_ops.split( - value=prec, num_or_size_splits=2, axis=0) - prec_low = array_ops.reshape(prec_low, shape=()) - prec_high = array_ops.reshape(prec_high, shape=()) - [rec_low, rec_high] = array_ops.split( - value=rec, num_or_size_splits=2, axis=0) - rec_low = array_ops.reshape(rec_low, shape=()) - rec_high = array_ops.reshape(rec_high, shape=()) + prec_low = prec[0] + prec_high = prec[1] + rec_low = rec[0] + rec_high = rec[1] sess.run(variables.local_variables_initializer()) sess.run([prec_op, rec_op]) @@ -2391,14 +2553,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.streaming_recall_at_thresholds( predictions, labels, thresholds, weights=weights) - [prec_low, prec_high] = array_ops.split( - value=prec, num_or_size_splits=2, axis=0) - prec_low = array_ops.reshape(prec_low, shape=()) - prec_high = array_ops.reshape(prec_high, shape=()) - [rec_low, rec_high] = array_ops.split( - value=rec, num_or_size_splits=2, axis=0) - rec_low = array_ops.reshape(rec_low, shape=()) - rec_high = array_ops.reshape(rec_high, shape=()) + prec_low = prec[0] + prec_high = prec[1] + rec_low = rec[0] + rec_high = rec[1] sess.run(variables.local_variables_initializer()) sess.run([prec_op, rec_op]) @@ -2420,10 +2578,10 @@ class StreamingPrecisionRecallThresholdsTest(test.TestCase): rec, rec_op = metrics.streaming_recall_at_thresholds(predictions, labels, thresholds) - [prec_low, prec_high] = array_ops.split( - value=prec, num_or_size_splits=2, axis=0) - [rec_low, rec_high] = array_ops.split( - value=rec, num_or_size_splits=2, axis=0) + prec_low = prec[0] + prec_high = prec[1] + rec_low = rec[0] + rec_high = rec[1] sess.run(variables.local_variables_initializer()) sess.run([prec_op, rec_op]) @@ -2562,7 +2720,7 @@ class StreamingFPRThresholdsTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) thresholds = [0, 0.5, 1.0] fpr, fpr_op = metrics.streaming_false_positive_rate_at_thresholds( predictions, labels, thresholds) @@ -2794,7 +2952,7 @@ class StreamingFNRThresholdsTest(test.TestCase): predictions = random_ops.random_uniform( (10, 3), maxval=1, dtype=dtypes_lib.float32, seed=1) labels = random_ops.random_uniform( - (10, 3), maxval=1, dtype=dtypes_lib.int64, seed=2) + (10, 3), maxval=2, dtype=dtypes_lib.int64, seed=2) thresholds = [0, 0.5, 1.0] fnr, fnr_op = metrics.streaming_false_negative_rate_at_thresholds( predictions, labels, thresholds) @@ -6012,5 +6170,163 @@ class AggregateMetricMapTest(test.TestCase): self.assertEqual(4, names_to_values['m2'].eval()) +class CountTest(test.TestCase): + + def setUp(self): + ops.reset_default_graph() + + def testVars(self): + metrics.count(array_ops.ones([4, 3])) + _assert_local_variables(self, ['count/count:0']) + + def testMetricsCollection(self): + my_collection_name = '__metrics__' + mean, _ = metrics.count( + array_ops.ones([4, 3]), metrics_collections=[my_collection_name]) + self.assertListEqual(ops.get_collection(my_collection_name), [mean]) + + def testUpdatesCollection(self): + my_collection_name = '__updates__' + _, update_op = metrics.count( + array_ops.ones([4, 3]), updates_collections=[my_collection_name]) + self.assertListEqual(ops.get_collection(my_collection_name), [update_op]) + + def testBasic(self): + with self.test_session() as sess: + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + result, update_op = metrics.count(values) + + sess.run(variables.local_variables_initializer()) + for _ in range(4): + sess.run(update_op) + self.assertAlmostEqual(8.0, sess.run(result), 5) + + def testUpdateOpsReturnsCurrentValue(self): + with self.test_session() as sess: + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + result, update_op = metrics.count(values) + + sess.run(variables.local_variables_initializer()) + + self.assertAlmostEqual(2.0, sess.run(update_op), 5) + self.assertAlmostEqual(4.0, sess.run(update_op), 5) + self.assertAlmostEqual(6.0, sess.run(update_op), 5) + self.assertAlmostEqual(8.0, sess.run(update_op), 5) + + self.assertAlmostEqual(8.0, sess.run(result), 5) + + def test1dWeightedValues(self): + with self.test_session() as sess: + # Create the queue that populates the values. + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 1)) + _enqueue_vector(sess, weights_queue, [0.5]) + _enqueue_vector(sess, weights_queue, [0]) + _enqueue_vector(sess, weights_queue, [0]) + _enqueue_vector(sess, weights_queue, [1.2]) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for _ in range(4): + update_op.eval() + self.assertAlmostEqual(3.4, result.eval(), 5) + + def test1dWeightedValues_placeholders(self): + with self.test_session() as sess: + # Create the queue that populates the values. + feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) + values = array_ops.placeholder(dtype=dtypes_lib.float32) + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1,)) + _enqueue_vector(sess, weights_queue, 0.5, shape=(1,)) + _enqueue_vector(sess, weights_queue, 0, shape=(1,)) + _enqueue_vector(sess, weights_queue, 0, shape=(1,)) + _enqueue_vector(sess, weights_queue, 1.2, shape=(1,)) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for i in range(4): + update_op.eval(feed_dict={values: feed_values[i]}) + self.assertAlmostEqual(3.4, result.eval(), 5) + + def test2dWeightedValues(self): + with self.test_session() as sess: + # Create the queue that populates the values. + values_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, values_queue, [0, 1]) + _enqueue_vector(sess, values_queue, [-4.2, 9.1]) + _enqueue_vector(sess, values_queue, [6.5, 0]) + _enqueue_vector(sess, values_queue, [-3.2, 4.0]) + values = values_queue.dequeue() + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(1, 2)) + _enqueue_vector(sess, weights_queue, [1.1, 1]) + _enqueue_vector(sess, weights_queue, [1, 0]) + _enqueue_vector(sess, weights_queue, [0, 1]) + _enqueue_vector(sess, weights_queue, [0, 0]) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for _ in range(4): + update_op.eval() + self.assertAlmostEqual(4.1, result.eval(), 5) + + def test2dWeightedValues_placeholders(self): + with self.test_session() as sess: + # Create the queue that populates the values. + feed_values = ((0, 1), (-4.2, 9.1), (6.5, 0), (-3.2, 4.0)) + values = array_ops.placeholder(dtype=dtypes_lib.float32) + + # Create the queue that populates the weighted labels. + weights_queue = data_flow_ops.FIFOQueue( + 4, dtypes=dtypes_lib.float32, shapes=(2,)) + _enqueue_vector(sess, weights_queue, [1.1, 1], shape=(2,)) + _enqueue_vector(sess, weights_queue, [1, 0], shape=(2,)) + _enqueue_vector(sess, weights_queue, [0, 1], shape=(2,)) + _enqueue_vector(sess, weights_queue, [0, 0], shape=(2,)) + weights = weights_queue.dequeue() + + result, update_op = metrics.count(values, weights) + + variables.local_variables_initializer().run() + for i in range(4): + update_op.eval(feed_dict={values: feed_values[i]}) + self.assertAlmostEqual(4.1, result.eval(), 5) + + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD index d6508362b8bf01468a43b26d6a0d0c9807b5967e..ed9fb64b954cc3dfec06936b479226a7def90008 100644 --- a/tensorflow/contrib/nccl/BUILD +++ b/tensorflow/contrib/nccl/BUILD @@ -71,10 +71,15 @@ tf_kernel_library( "kernels/nccl_manager.cc", "kernels/nccl_manager.h", "kernels/nccl_ops.cc", + "kernels/nccl_rewrite.cc", ], deps = [ + "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:gpu_headers_lib", + "//tensorflow/core:lib", + "//tensorflow/core:proto_text", + "//tensorflow/core:stream_executor", "@nccl_archive//:nccl", ], alwayslink = 1, @@ -110,7 +115,11 @@ tf_custom_op_py_library( deps = [ ":nccl_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:device", + "//tensorflow/python:framework_ops", "//tensorflow/python:platform", + "//tensorflow/python:util", + "//tensorflow/python/eager:context", ], ) diff --git a/tensorflow/contrib/nccl/kernels/nccl_ops.cc b/tensorflow/contrib/nccl/kernels/nccl_ops.cc index 4eb52492dbcc386941029709631314634c1c9be1..266d4f6f0de0274dca2bfc9022503f09b0ca7d42 100644 --- a/tensorflow/contrib/nccl/kernels/nccl_ops.cc +++ b/tensorflow/contrib/nccl/kernels/nccl_ops.cc @@ -15,8 +15,6 @@ limitations under the License. #if GOOGLE_CUDA -#include -#include #include #include "src/nccl.h" @@ -24,6 +22,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" namespace tensorflow { +namespace { // Base class for all communicator ops that use nccl. // @@ -134,7 +133,7 @@ class NcclReduceSendKernel : public NcclReduceOpBase { compute_stream, &c->input(0), std::move(actual_done)); } }; -REGISTER_KERNEL_BUILDER(Name("NcclReduceSend").Device(DEVICE_GPU), +REGISTER_KERNEL_BUILDER(Name("_NcclReduceSend").Device(DEVICE_GPU), NcclReduceSendKernel); // To execute a single reduce, this kernel is called once for one devices, and @@ -166,7 +165,7 @@ class NcclReduceRecvKernel : public NcclReduceOpBase { private: ncclRedOp_t reduction_op_; }; -REGISTER_KERNEL_BUILDER(Name("NcclReduceRecv").Device(DEVICE_GPU), +REGISTER_KERNEL_BUILDER(Name("_NcclReduceRecv").Device(DEVICE_GPU), NcclReduceRecvKernel); // To execute a single broadcast, this kernel is called once for one device, and @@ -191,7 +190,7 @@ class NcclBroadcastSendKernel : public NcclAsyncOpBase { std::move(actual_done)); } }; -REGISTER_KERNEL_BUILDER(Name("NcclBroadcastSend").Device(DEVICE_GPU), +REGISTER_KERNEL_BUILDER(Name("_NcclBroadcastSend").Device(DEVICE_GPU), NcclBroadcastSendKernel); // To execute a single broadcast, this kernel is called once for all but one of @@ -206,7 +205,7 @@ class NcclBroadcastRecvKernel : public NcclAsyncOpBase { const Tensor& shape_t = c->input(0); TensorShape shape; OP_REQUIRES_OK_ASYNC( - c, TensorShapeUtils::MakeShape(shape_t.vec(), &shape), done); + c, TensorShapeUtils::MakeShape(shape_t.vec(), &shape), done); Tensor* out_t; OP_REQUIRES_OK_ASYNC(c, c->allocate_output(0, shape, &out_t), done); @@ -224,9 +223,24 @@ class NcclBroadcastRecvKernel : public NcclAsyncOpBase { } }; REGISTER_KERNEL_BUILDER( - Name("NcclBroadcastRecv").Device(DEVICE_GPU).HostMemory("shape"), + Name("_NcclBroadcastRecv").Device(DEVICE_GPU).HostMemory("shape"), NcclBroadcastRecvKernel); +// Define stub kernels for the ops that get replaced post placement. +class NcclStubKernel : public AsyncOpKernel { + public: + explicit NcclStubKernel(OpKernelConstruction* c) : AsyncOpKernel(c) {} + void ComputeAsync(OpKernelContext* c, DoneCallback done) override { + c->SetStatus(errors::Unimplemented( + "This op should be replaced during graph optimization.")); + done(); + } +}; +REGISTER_KERNEL_BUILDER(Name("NcclBroadcast").Device(DEVICE_GPU), + NcclStubKernel); +REGISTER_KERNEL_BUILDER(Name("NcclReduce").Device(DEVICE_GPU), NcclStubKernel); + +} // namespace } // namespace tensorflow #endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc new file mode 100644 index 0000000000000000000000000000000000000000..a4de46a93fab1dfe93b47f2789cc533bc447e43a --- /dev/null +++ b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc @@ -0,0 +1,276 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#include +#include + +#include "tensorflow/core/common_runtime/optimization_registry.h" +#include "tensorflow/core/graph/node_builder.h" + +namespace tensorflow { +namespace { + +// Replaces NcclReduce node with _NcclReduceRecv reusing one input of same +// device, adds one _NcclReduceSend for each other input. +Status ReplaceReduce(Graph* graph, Node* node) { + string reduction; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "reduction", &reduction)); + DataType dtype; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype)); + int num_devices = node->num_inputs(); + string shared_name = node->name(); + auto make_builder = [&](StringPiece op_name, StringPiece suffix) { + return NodeBuilder(strings::StrCat(shared_name, suffix), op_name) + .Attr("reduction", reduction) + .Attr("num_devices", num_devices) + .Attr("shared_name", shared_name) + .Attr("T", dtype); + }; + std::vector control_inputs; + for (const auto& edge : node->in_edges()) { + if (edge->IsControlEdge()) { + control_inputs.push_back(edge->src()); + } + } + std::vector out_nodes; + for (const auto& edge : node->out_edges()) { + out_nodes.emplace_back(edge->dst(), edge->dst_input()); + } + int recv_dev = node->assigned_device_name_index(); + NodeBuilder recv_builder = + make_builder("_NcclReduceRecv", "Recv").ControlInputs(control_inputs); + bool recv_input_set = false; + int send_counter = 0; + for (const auto& edge : node->in_edges()) { + Node* src_node = edge->src(); + if (edge->IsControlEdge()) { + continue; + } + int send_dev = src_node->assigned_device_name_index(); + if (!recv_input_set && send_dev == recv_dev) { + recv_builder.Input(src_node); + recv_input_set = true; + continue; + } + auto send_builder = make_builder("_NcclReduceSend", + strings::StrCat("Send_", ++send_counter)) + .Input(src_node) + .ControlInputs(control_inputs); + Node* send_node = nullptr; + TF_RETURN_IF_ERROR(send_builder.Finalize(graph, &send_node)); + send_node->set_assigned_device_name_index(send_dev); + // Send nodes don't have any outputs and therefore have no data dependencies + // to the outputs of the graph. We add a control dependency to the receive + // node so that those 'dangling' nodes are run. + // TODO(b/67027412): Avoid these cross-device control edges. + for (const auto& out_node : out_nodes) { + graph->AddControlEdge(send_node, out_node.node); + } + } + if (!recv_input_set) { + return errors::InvalidArgument( + "No input tensor uses the same device as the NcclReduce op"); + } + Node* recv_node = nullptr; + TF_RETURN_IF_ERROR(recv_builder.Finalize(graph, &recv_node)); + recv_node->set_assigned_device_name_index(recv_dev); + graph->RemoveNode(node); + for (const auto& out_node : out_nodes) { + if (out_node.index == Graph::kControlSlot) { + graph->AddControlEdge(recv_node, out_node.node); + } else { + graph->AddEdge(recv_node, 0, out_node.node, out_node.index); + } + } + return Status::OK(); +} + +TensorProto TensorFromShape(const TensorShapeProto& shape) { + TensorProto result; + result.set_dtype(DT_INT32); + for (const auto& dim : shape.dim()) { + result.add_int_val(dim.size()); + } + result.mutable_tensor_shape()->add_dim()->set_size(shape.dim_size()); + return result; +} + +// Replaces NcclBroadcast node with _NcclBroadcastSend, connects the input to +// all outputs of same device, adds one _NcclBroadcastRecv for each other output +// device. +Status ReplaceBroadcast(Graph* graph, Node* node) { + DataType dtype; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "T", &dtype)); + int send_dev = node->assigned_device_name_index(); + int num_devices = 0; // Number of distinct devices, incremented below. + std::vector recv_index_map; // Map device name index to stable index. + + // Map device name index to nodes that take the broadcast as input. + std::vector> out_nodes_map; + for (const auto& edge : node->out_edges()) { + int dst_dev = edge->IsControlEdge() + ? send_dev + : edge->dst()->assigned_device_name_index(); + if (out_nodes_map.size() <= dst_dev) { + out_nodes_map.resize(dst_dev + 1); + recv_index_map.resize(dst_dev + 1); + } + auto it = out_nodes_map.begin() + dst_dev; + if (it->empty()) { + recv_index_map[dst_dev] = num_devices; + ++num_devices; + } + it->emplace_front(NodeBuilder::NodeOut(edge->dst(), edge->dst_input())); + } + + if (num_devices <= 1) { + // Only one participating device, skip NCCL op. + const Edge* in_edge = nullptr; + TF_RETURN_IF_ERROR(node->input_edge(0, &in_edge)); + Node* in_node = in_edge->src(); + int in_index = in_edge->src_output(); + graph->RemoveNode(node); + for (const auto& out_nodes : out_nodes_map) { + for (const auto& out_node : out_nodes) { + if (out_node.index == Graph::kControlSlot) { + graph->AddControlEdge(in_node, out_node.node); + } else { + graph->AddEdge(in_node, in_index, out_node.node, out_node.index); + } + } + } + return Status::OK(); + } + + string shared_name = node->name(); + auto make_builder = [&](StringPiece op_name, StringPiece suffix) { + return NodeBuilder(strings::StrCat(shared_name, suffix), op_name) + .Attr("num_devices", num_devices) + .Attr("shared_name", shared_name) + .Attr("T", dtype); + }; + + // Create broadcast send node and replace the original broadcast node. + NodeBuilder::NodeOut in_node; + NodeBuilder send_builder = make_builder("_NcclBroadcastSend", "Send"); + for (const auto& edge : node->in_edges()) { + if (edge->IsControlEdge()) { + send_builder.ControlInput(edge->src()); + } else { + in_node = NodeBuilder::NodeOut(edge->src(), edge->src_output()); + send_builder.Input(in_node); + } + } + Node* send_node = nullptr; + TF_RETURN_IF_ERROR(send_builder.Finalize(graph, &send_node)); + send_node->set_assigned_device_name_index(send_dev); + + TensorShapeProto shape_proto; + TF_RETURN_IF_ERROR(GetNodeAttr(node->attrs(), "shape", &shape_proto)); + + // Delete the original node before reconnecting to outputs. + graph->RemoveNode(node); + + // Connect all outputs on the device of broadcast send. + for (const auto& out_node : out_nodes_map[send_dev]) { + if (out_node.index == Graph::kControlSlot) { + graph->AddControlEdge(send_node, out_node.node); + } else { + graph->AddEdge(in_node.node, in_node.index, out_node.node, + out_node.index); + // Add control edge so send node is run. + graph->AddControlEdge(send_node, out_node.node); + } + } + out_nodes_map[send_dev].clear(); + + TensorProto tensor_proto = TensorFromShape(shape_proto); + bool is_fully_defined = TensorShape(shape_proto).IsFullyDefined(); + string shape_name = strings::StrCat(in_node.node->name(), "/Shape"); + Node* shape_node = nullptr; + if (!is_fully_defined) { + NodeBuilder shape_builder(shape_name, "Shape"); + shape_builder.Input(in_node).Attr("out_type", DT_INT32).Attr("T", dtype); + TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node)); + shape_node->set_assigned_device_name_index(send_dev); + } + + // For all other devices, create a broadcast receive and connect outputs. + for (int recv_dev = 0; recv_dev < out_nodes_map.size(); ++recv_dev) { + if (out_nodes_map[recv_dev].empty()) { + continue; + } + int recv_index = recv_index_map[recv_dev]; + if (is_fully_defined) { + // If the shape is fully defined, define one const node per device. + NodeBuilder shape_builder(strings::StrCat(shape_name, recv_index), + "Const"); + shape_builder.Attr("value", tensor_proto).Attr("dtype", DT_INT32); + TF_RETURN_IF_ERROR(shape_builder.Finalize(graph, &shape_node)); + shape_node->set_assigned_device_name_index(recv_dev); + } + Node* recv_node; + TF_RETURN_IF_ERROR( + make_builder("_NcclBroadcastRecv", strings::StrCat("Recv_", recv_index)) + .Input(shape_node) + .Finalize(graph, &recv_node)); + recv_node->set_assigned_device_name_index(recv_dev); + for (const auto& out_node : out_nodes_map[recv_dev]) { + graph->AddEdge(recv_node, 0, out_node.node, out_node.index); + } + } + + return Status::OK(); +} + +// Replaces occurrences of Nccl{Reduce, Broadcast}Input/Output with their +// _Nccl...Send/Recv counterparts and removes data dependencies between them. +class NcclReplacePass : public GraphOptimizationPass { + public: + Status Run(const GraphOptimizationPassOptions& options) override { + if (options.graph == nullptr) { + return Status::OK(); + } + Graph* graph = options.graph->get(); + if (graph == nullptr) { + return errors::Internal( + "NCCL replacement should happen before partitioning and a " + "graph should be available."); + } + // Find reduction and broadcast ops and replace them with Send/Recv ops. + for (Node* node : graph->op_nodes()) { + StringPiece type = node->type_string(); + if (!type.starts_with("Nccl")) { + continue; + } + if (type == "NcclReduce") { + TF_RETURN_IF_ERROR(ReplaceReduce(graph, node)); + } + if (type == "NcclBroadcast") { + TF_RETURN_IF_ERROR(ReplaceBroadcast(graph, node)); + } + } + return Status::OK(); + } +}; +REGISTER_OPTIMIZATION(OptimizationPassRegistry::POST_PLACEMENT, 0, + NcclReplacePass); + +} // namespace +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/contrib/nccl/ops/nccl_ops.cc b/tensorflow/contrib/nccl/ops/nccl_ops.cc index 532c79c24cc9596af580ee3faf463aecbc59bb07..8eb804c2e988f313ba1b340217cae20f1f5502c7 100644 --- a/tensorflow/contrib/nccl/ops/nccl_ops.cc +++ b/tensorflow/contrib/nccl/ops/nccl_ops.cc @@ -45,7 +45,28 @@ num_devices: The number of devices participating in this reduction. shared_name: Identifier that shared between ops of the same reduction. )doc"); -REGISTER_OP("NcclReduceSend") +// Note: This op has no kernel implementation, but is replaced by +// _NcclReduceSend and _NcclReduceRecv during graph optimization stage. +REGISTER_OP("NcclReduce") + .Input("input: num_devices * T") + .Output("data: T") + .Attr("reduction: {'min', 'max', 'prod', 'sum'}") + .Attr("T: {float, float64, int32, int64}") + .Attr("num_devices: int") + .SetIsStateful() + .SetShapeFn(shape_inference::UnchangedShape) + .Doc(R"doc( +Reduces `input` from `num_devices` using `reduction` to a single device. + +The graph should be constructed so that all inputs have a valid device +assignment, and the op itself is assigned one of these devices. + +input: The input to the reduction. +data: the value of the reduction across all `num_devices` devices. +reduction: the reduction operation to perform. + )doc"); + +REGISTER_OP("_NcclReduceSend") .Input("input: T") .Attr("reduction: {'min', 'max', 'prod', 'sum'}") .Attr("T: {float, float64, int32, int64}") @@ -54,19 +75,20 @@ REGISTER_OP("NcclReduceSend") .SetIsStateful() .SetShapeFn(shape_inference::NoOutputs) .Doc(R"doc( -Reduces `input` to the NcclReduceRecv op registered in the same `shared_name`. +Replacement node for NcclReduce. +Reduces `input` to the NcclReduceRecv op registered in the same `shared_name`. The graph should be constructed so that 'num_devices-1' devices run -`NcclReduceSend` and one device runs NcclReduceRecv op with shared_name value +`_NcclReduceSend` and one device runs _NcclReduceRecv op with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. -input: The input to the reduction +input: The input to the reduction. reduction: the reduction operation to perform. num_devices: The number of devices participating in this reduction. shared_name: Identifier that is shared between ops of the same reduce. )doc"); -REGISTER_OP("NcclReduceRecv") +REGISTER_OP("_NcclReduceRecv") .Input("input: T") .Output("data: T") .Attr("reduction: {'min', 'max', 'prod', 'sum'}") @@ -76,21 +98,42 @@ REGISTER_OP("NcclReduceRecv") .SetIsStateful() .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( +Replacement node for NcclReduce. + Reduces 'input' from this op and the NcclReduceSend ops registered in the same `shared_name`. - The graph should be constructed so that 'num_devices-1' devices run -`NcclReduceSend` and one device runs NcclReduceRecv op with shared_name value +`_NcclReduceSend` and one device runs _NcclReduceRecv op with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. -input: The input to the reduction +input: The input to the reduction. data: The reduced data received from this op and the NcclReduceSend op. reduction: the reduction operation to perform. num_devices: The number of devices participating in this reduction. shared_name: Identifier that is shared between ops of the same reduce. )doc"); -REGISTER_OP("NcclBroadcastSend") +// Note: This op has no kernel implementation, but is replaced by +// _NcclBroadcastSend and _NcclBroadcastRecv during graph optimization stage. +REGISTER_OP("NcclBroadcast") + .Input("input: T") + .Output("output: T") + .Attr("T: {float, float64, int32, int64}") + .Attr("shape: shape") + .SetIsStateful() + .SetShapeFn(shape_inference::UnchangedShape) + .Doc(R"doc( +Sends `input` to all devices that are connected to the output. + +The graph should be constructed so that all ops connected to the output have a +valid device assignment, and the op itself is assigned one of these devices. + +input: The input to the broadcast. +output: The same as input. +shape: The shape of the input tensor. + )doc"); + +REGISTER_OP("_NcclBroadcastSend") .Input("input: T") .Attr("T: {float, float64, int32, int64}") .Attr("num_devices: int") @@ -98,19 +141,21 @@ REGISTER_OP("NcclBroadcastSend") .SetIsStateful() .SetShapeFn(shape_inference::NoOutputs) .Doc(R"doc( -Sends `input` to the NcclBroadcastRecv ops registered in the same `shared_name`. +Replacement node for NcclBroadcast. -The graph should be constructed so that one device runs `NcclBroadcastSend` and -`num_devices-1` devices run NcclBroadcastRecv ops with shared_name value `c`. +Sends `input` to the _NcclBroadcastRecv ops registered in the same +`shared_name`. +The graph should be constructed so that one device runs `_NcclBroadcastSend` and +`num_devices-1` devices run _NcclBroadcastRecv ops with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. -input: The input to the broadcast +input: The input to the broadcast. num_devices: The number of devices participating in this reduction. shared_name: Identifier that is shared between ops of the same broadcast. )doc"); -REGISTER_OP("NcclBroadcastRecv") - .Input("shape: int64") +REGISTER_OP("_NcclBroadcastRecv") + .Input("shape: int32") .Output("output: T") .Attr("T: {float, float64, int32, int64}") .Attr("num_devices: int") @@ -123,11 +168,12 @@ REGISTER_OP("NcclBroadcastRecv") return Status::OK(); }) .Doc(R"doc( -Sends data of shape `shape` from the NcclBroadcastSend op registered in the -same `shared_name`. +Replacement node for NcclBroadcast. -The graph should be constructed so that one device runs `NcclBroadcastSend` and -`num_devices-1` devices run NcclBroadcastRecv ops with shared_name value `c`. +Sends data of shape `shape` from the _NcclBroadcastSend op registered in the +same `shared_name`. +The graph should be constructed so that one device runs `_NcclBroadcastSend` and +`num_devices-1` devices run _NcclBroadcastRecv ops with shared_name value `c`. Failure to do so will cause the graph execution to fail to complete. shape: The shape of the output. diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops.py b/tensorflow/contrib/nccl/python/ops/nccl_ops.py index 906d9f948acf212dce1dbbbf9ec7c60c30f389b1..8dc038b9ac992de7db8b762e3697c6693099e192 100644 --- a/tensorflow/contrib/nccl/python/ops/nccl_ops.py +++ b/tensorflow/contrib/nccl/python/ops/nccl_ops.py @@ -23,9 +23,7 @@ from tensorflow.contrib.nccl.ops import gen_nccl_ops from tensorflow.contrib.util import loader from tensorflow.python.eager import context from tensorflow.python.framework import device -from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops from tensorflow.python.platform import resource_loader _nccl_ops_so = loader.load_op_library( @@ -64,13 +62,13 @@ def _all_sum_grad(op, grad): LookupError: If `reduction` is not `sum`. """ if op.get_attr('reduction') != 'sum': - raise LookupError('No gradient defined for NcclAllReduce except all_sum.') + raise LookupError('No gradient defined for NcclAllReduce except sum.') - _check_device_assignment(grad) + _check_device(grad, expected=op.device) num_devices = op.get_attr('num_devices') shared_name = op.get_attr('shared_name') + '_grad' - with ops.device(grad.device): + with ops.device(op.device): return gen_nccl_ops.nccl_all_reduce( input=grad, reduction='sum', @@ -129,7 +127,7 @@ def all_max(tensors): return _apply_all_reduce('max', tensors) -def reduce_sum(tensors, dst_device): +def reduce_sum(tensors): """Returns a tensor with the reduce sum across `tensors`. The computation is done with a reduce operation, so only one tensor is @@ -138,54 +136,76 @@ def reduce_sum(tensors, dst_device): Args: tensors: The input tensors across which to sum; must be assigned to GPU devices. - dst_device: The device of the returned tensor. Returns: - A tensor containing the sum of the input tensors, with the device of the - tensor being `dst_device`. + A tensor containing the sum of the input tensors. + + Raises: + LookupError: If context is not currently using a GPU device. + """ + return _apply_reduce('sum', tensors) + + +@ops.RegisterGradient('NcclReduce') +def _reduce_sum_grad(op, grad): + """The gradients for input `Operation` of `reduce_sum`. + + Args: + op: The `sum send` `Operation` that we are differentiating. + grad: Gradient with respect to the output of the `reduce_sum` op. + + Returns: + The gradient with respect to the input of `reduce_sum` op. + + Raises: + LookupError: If the reduction attribute of op is not `sum`. """ - return _apply_reduce('sum', tensors, dst_device) + if op.get_attr('reduction') != 'sum': + raise LookupError('No gradient defined for NcclReduce except sum.') + _check_device(grad, expected=op.device) + with ops.device(op.device): + result = gen_nccl_ops.nccl_broadcast(input=grad, shape=grad.shape) -def broadcast(src_tensor, dst_devices): - """Returns a list of tensors on `dst_devices`, each with value `tensor`. + return [result] * len(op.inputs) - The computation is done with a broadcast nccl operation, so if only some of - the returned tensors and src_tensor are evaluated then the computation will - hang. + +def broadcast(tensor): + """Returns a tensor that can be efficiently transferred to other devices. Args: - src_tensor: The tensor to send; must be assigned to a GPU device. - dst_devices: The GPU devices to receive the sent tensor. + tensor: The tensor to send; must be assigned to a GPU device. Returns: - An `Operation` to send the `src_tensor`, and a list of tensors, each with - the value of `src_tensor`, where the device of tensor i is `dst_devices[i]`. + A tensor with the value of `src_tensor`, which can be used as input to + ops on other GPU devices. """ - if not dst_devices: - raise ValueError('Must pass >0 dst_devices to broadcast') _check_graph_mode() - _check_device_assignment(src_tensor) + _check_device(tensor) - shape = array_ops.shape(src_tensor, out_type=dtypes.int64) - num_devices = len(dst_devices) + 1 - shared_name = _get_shared_name() + with ops.device(tensor.device): + return gen_nccl_ops.nccl_broadcast(input=tensor, shape=tensor.shape) - with ops.device(src_tensor.device): - send = gen_nccl_ops.nccl_broadcast_send( - input=src_tensor, num_devices=num_devices, shared_name=shared_name) - - recvs = [] - for d in dst_devices: - with ops.device(d): - recvs.append( - gen_nccl_ops.nccl_broadcast_recv( - shape=shape, - T=src_tensor.dtype, - num_devices=num_devices, - shared_name=shared_name)) - return send, recvs +@ops.RegisterGradient('NcclBroadcast') +def _broadcast_grad(op, accumulated_grad): + """The gradients for input `Operation` of `broadcast`. + + Args: + op: The `broadcast send` `Operation` that we are differentiating. + accumulated_grad: Accumulated gradients with respect to the output of the + `broadcast` op. + + Returns: + Gradients with respect to the input of `broadcast`. + """ + # Grab inputs of accumulated_grad and replace accumulation with reduce_sum. + grads = [t for t in accumulated_grad.op.inputs] + for t in grads: + _check_device(t) + + with ops.device(op.device): + return gen_nccl_ops.nccl_reduce(input=grads, reduction='sum') def _apply_all_reduce(reduction, tensors): @@ -198,7 +218,7 @@ def _apply_all_reduce(reduction, tensors): res = [] for t in tensors: - _check_device_assignment(t) + _check_device(t) with ops.device(t.device): res.append( gen_nccl_ops.nccl_all_reduce( @@ -210,40 +230,20 @@ def _apply_all_reduce(reduction, tensors): return res -def _apply_reduce(reduction, tensors, dst_device): +def _apply_reduce(reduction, tensors): """Helper function for reduce_* functions.""" if not tensors: raise ValueError('Must pass >0 tensors to reduce operations') - if not dst_device: - raise ValueError('Must pass dst_device to reduce operations') _check_graph_mode() + for t in tensors: + _check_device(t) + result = gen_nccl_ops.nccl_reduce(input=tensors, reduction=reduction) try: - recv_index = next(i for i, t in enumerate(tensors) - if t.device == dst_device) + next(t for t in tensors if t.device == result.device) except StopIteration: - raise ValueError('One of the tensors must be assigned to dst_device') - shared_name = _get_shared_name() - - sends = [] - for t in tensors[:recv_index] + tensors[recv_index + 1:]: - _check_device_assignment(t) - with ops.device(t.device): - sends.append( - gen_nccl_ops.nccl_reduce_send( - input=t, - reduction=reduction, - num_devices=len(tensors), - shared_name=shared_name)) - - with ops.device(dst_device): - recv = gen_nccl_ops.nccl_reduce_recv( - input=tensors[recv_index], - reduction=reduction, - num_devices=len(tensors), - shared_name=shared_name) - - return recv, sends + raise ValueError('One input tensor must be assigned to current device') + return result _lock = threading.Lock() @@ -259,9 +259,11 @@ def _get_shared_name(): return 'c%s' % val -def _check_device_assignment(tensor): +def _check_device(tensor, expected=None): if not device.canonical_name(tensor.device): raise ValueError('Device assignment required for nccl collective ops') + if expected and expected != tensor.device: + raise ValueError('Expected device %s, got %s' % (expected, tensor.device)) def _check_graph_mode(): diff --git a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py index 96d67723a0ad197436a12924bd2b4ecb73eee4cb..0b13e3595e36b609468f459d9179f8e9f5c1e055 100644 --- a/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py +++ b/tensorflow/contrib/nccl/python/ops/nccl_ops_test.py @@ -22,8 +22,10 @@ from functools import partial import numpy as np from tensorflow.contrib import nccl +from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradients from tensorflow.python.platform import test @@ -36,27 +38,30 @@ def _DeviceTensors(tensors, devices): def _NcclAllReduce(nccl_fun, tensors, devices): - return nccl_fun(_DeviceTensors(tensors, devices)), [] + return nccl_fun(_DeviceTensors(tensors, devices)) def _NcclReduce(nccl_fun, tensors, devices): - d_tensors = _DeviceTensors(tensors, devices) receiver = np.random.randint(0, len(devices)) - received_tensor, send_ops = nccl_fun(d_tensors, devices[receiver]) - return [received_tensor], send_ops + with ops.device(devices[receiver]): + return [nccl_fun(_DeviceTensors(tensors, devices))] def _NcclBroadcast(tensors, devices): sender = np.random.randint(0, len(devices)) - d_tensor = _DeviceTensors(tensors[0:1], devices[sender:sender + 1])[0] - other_devices = devices[:sender] + devices[sender + 1:] - send_op, received_tensors = nccl.broadcast(d_tensor, other_devices) - return received_tensors, [send_op] + with ops.device(devices[sender]): + tensor = array_ops.identity(tensors[0]) + broadcast = nccl.broadcast(tensor) + return _DeviceTensors([broadcast] * len(devices), devices) class NcclTestCase(test.TestCase): - def _Test(self, nccl_reduce, numpy_fn): + def _Test(self, + nccl_reduce, + numpy_fn, + device_sets=(['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'], + ['/device:GPU:1', '/device:GPU:0'])): """Tests that nccl_reduce does the same as reduction with numpy_fn. Args: @@ -65,6 +70,7 @@ class NcclTestCase(test.TestCase): reduction. numpy_fn: A function taking two tensors and returning the reduction of the two. + device_sets: Tuple of virtual devices to run test on. """ if not test.is_gpu_available(): return # Test requires access to a GPU @@ -74,26 +80,28 @@ class NcclTestCase(test.TestCase): # same communicator across multiple sessions. with self.test_session(use_gpu=True) as sess: - for devices in [['/device:GPU:1', '/device:GPU:2', '/device:GPU:0'], - ['/device:GPU:1', '/device:GPU:0']]: + for devices in device_sets: shape = (3, 4) random = (np.random.random_sample(shape) - .5) * 1024 - tensors = [random.astype(dtype)] * len(devices) + tensors = [] + for _ in devices: + tensors.append(random.astype(dtype)) np_ans = tensors[0] for t in tensors[1:]: np_ans = numpy_fn(np_ans, t) - reduce_tensors, reduce_ops = nccl_reduce(tensors, devices) + reduce_tensors = nccl_reduce(tensors, devices) self.assertNotEmpty(reduce_tensors) # Test shape inference. for r in reduce_tensors: self.assertEqual(shape, r.get_shape()) + result_tensors = [array_ops.identity(t) for t in reduce_tensors] + # Test execution and results. - nccl_results = sess.run(reduce_tensors + reduce_ops) - for r in nccl_results[:len(reduce_tensors)]: - self.assertAllClose(r, np_ans) + for t in sess.run(result_tensors): + self.assertAllClose(t, np_ans) def _TestGradient(self, nccl_reduce, numpy_fn): """Tests the gradient of nccl_reduce. @@ -106,14 +114,12 @@ class NcclTestCase(test.TestCase): reduction of the two. """ def _Gradient(tensors, devices): - reduce_tensors, _ = nccl_reduce(tensors, devices) - tensor_ops = [t.op for t in reduce_tensors] - d_tensors = _DeviceTensors(tensors, devices) - grad_tensors = [ - ops.get_gradient_function(op)(op, loss) - for op, loss in zip(tensor_ops, d_tensors) - ] - return grad_tensors, [] + inputs = [array_ops.placeholder(t.dtype, t.shape) for t in tensors] + reduce_tensors = nccl_reduce(inputs, devices) + losses = _DeviceTensors(tensors, [t.device for t in reduce_tensors]) + grads = gradients.gradients( + reduce_tensors, inputs, losses, colocate_gradients_with_ops=True) + return [g for g in grads if g is not None] self._Test(_Gradient, numpy_fn) @@ -142,27 +148,40 @@ class SingleReduceTest(NcclTestCase): def testSum(self): self._Test(partial(_NcclReduce, nccl.reduce_sum), lambda x, y: x + y) + def testSumGrad(self): + self._TestGradient(partial(_NcclReduce, nccl.reduce_sum), lambda x, y: x) + class BroadcastTest(NcclTestCase): def testBroadcast(self): self._Test(_NcclBroadcast, lambda x, y: x) + def testBroadcastSingleDevice(self): + # Broadcasts on a single device are removed completely during rewrite. + self._Test(_NcclBroadcast, lambda x, y: x, + (['/device:GPU:0', '/device:GPU:0'],)) + + def testBroadcastToCpuError(self): + # Broadcasts to CPU is not supported. + with self.assertRaisesRegexp( + errors.NotFoundError, + "No registered '_NcclBroadcastRecv' OpKernel for CPU devices"): + self._Test(_NcclBroadcast, lambda x, y: x, + (['/device:GPU:0', '/device:CPU:0'],)) + class CombinedTest(NcclTestCase): """Test all-reduce vs. single-reduce plus broadcast in one session.run.""" - def _combined(self, tensors, devices): - all_reduce_tensors = _NcclAllReduce(nccl.all_sum, tensors, devices)[0] - single_reduce_tensors, single_reduce_ops = _NcclReduce( - nccl.reduce_sum, tensors, devices) - broadcast_tensors, broadcast_ops = _NcclBroadcast(single_reduce_tensors, - devices) - all_tensors = all_reduce_tensors + single_reduce_tensors + broadcast_tensors - return all_tensors, single_reduce_ops + broadcast_ops + def _Combined(self, tensors, devices): + all_reduce_tensors = _NcclAllReduce(nccl.all_sum, tensors, devices) + single_reduce_tensors = _NcclReduce(nccl.reduce_sum, tensors, devices) + broadcast_tensors = _NcclBroadcast(single_reduce_tensors, devices) + return all_reduce_tensors + broadcast_tensors def testCombined(self): - self._Test(self._combined, lambda x, y: x + y) + self._Test(self._Combined, lambda x, y: x + y) if __name__ == '__main__': diff --git a/tensorflow/contrib/nearest_neighbor/BUILD b/tensorflow/contrib/nearest_neighbor/BUILD index 84d59cc4be87488ec55df54af16ae0b27a37fdd0..9500c18b1df9d772dfb827bc2b3d33e0a65974f6 100644 --- a/tensorflow/contrib/nearest_neighbor/BUILD +++ b/tensorflow/contrib/nearest_neighbor/BUILD @@ -41,18 +41,14 @@ tf_gen_op_wrapper_py( tf_custom_op_py_library( name = "nearest_neighbor_py", srcs = ["__init__.py"] + glob(["python/ops/*.py"]), - dso = [ - ":python/ops/_nearest_neighbor_ops.so", - ], - kernels = [ - ":nearest_neighbor_ops_kernels", - ], + dso = [":python/ops/_nearest_neighbor_ops.so"], + kernels = [":nearest_neighbor_ops_kernels"], srcs_version = "PY2AND3", visibility = ["//visibility:public"], deps = [ - ":nearest_neighbor_ops_pywrapper", "//tensorflow/contrib/util:util_py", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", ], ) @@ -70,9 +66,7 @@ tf_kernel_library( cc_library( name = "heap", - hdrs = [ - "kernels/heap.h", - ], + hdrs = ["kernels/heap.h"], ) tf_cc_test( @@ -81,17 +75,14 @@ tf_cc_test( srcs = ["kernels/heap_test.cc"], deps = [ ":heap", - "//tensorflow/core:test", "//tensorflow/core:test_main", - "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", ], ) cc_library( name = "hyperplane_lsh_probes", - hdrs = [ - "kernels/hyperplane_lsh_probes.h", - ], + hdrs = ["kernels/hyperplane_lsh_probes.h"], deps = [ ":heap", "//third_party/eigen3", @@ -107,6 +98,7 @@ tf_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", ], ) diff --git a/tensorflow/contrib/nn/BUILD b/tensorflow/contrib/nn/BUILD index 0ed7e521596c256293c3432ff6b9225e0beae709..56a24ac77f0b9a87b6e4db48cddacdf35f4855d0 100644 --- a/tensorflow/contrib/nn/BUILD +++ b/tensorflow/contrib/nn/BUILD @@ -30,6 +30,7 @@ py_library( "//tensorflow/python:function", "//tensorflow/python:math_ops", "//tensorflow/python:nn", + "//tensorflow/python:nn_ops", "//tensorflow/python:random_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:tensor_util", @@ -77,9 +78,9 @@ py_test( deps = [ ":nn_py", "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", - "//tensorflow/python:framework_ops", - "//tensorflow/python:nn", + "//tensorflow/python:constant_op", + "//tensorflow/python:gradient_checker", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/nn/__init__.py b/tensorflow/contrib/nn/__init__.py index 7007e26baced68db0d3ccdfc8145a95724139391..3bf795d19aad73ec37c0485fe1900a7d8ac43137 100644 --- a/tensorflow/contrib/nn/__init__.py +++ b/tensorflow/contrib/nn/__init__.py @@ -18,6 +18,7 @@ @@deprecated_flipped_softmax_cross_entropy_with_logits @@deprecated_flipped_sparse_softmax_cross_entropy_with_logits @@deprecated_flipped_sigmoid_cross_entropy_with_logits +@@nth_element @@rank_sampled_softmax_loss @@scaled_softplus """ @@ -31,6 +32,7 @@ from tensorflow.contrib.nn.python.ops.alpha_dropout import * from tensorflow.contrib.nn.python.ops.cross_entropy import * from tensorflow.contrib.nn.python.ops.sampling_ops import * from tensorflow.contrib.nn.python.ops.scaled_softplus import * +from tensorflow.python.ops.nn_ops import nth_element # pylint: enable=unused-import,wildcard-import from tensorflow.python.util.all_util import remove_undocumented diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD index b5a67206f3433ab3cf5ee5594557aadf8a09983b..096d2270e4c2d046a8dc8982bf03a648a195c667 100644 --- a/tensorflow/contrib/opt/BUILD +++ b/tensorflow/contrib/opt/BUILD @@ -86,9 +86,9 @@ py_test( ], deps = [ ":opt_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variables", "//third_party/py/numpy", @@ -119,13 +119,13 @@ py_test( deps = [ ":opt_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:session", "//tensorflow/python:variables", "//third_party/py/numpy", ], @@ -139,12 +139,17 @@ tf_py_test( "//third_party/py/numpy", "//tensorflow/python:client", "//tensorflow/python:client_testlib", + "//tensorflow/python:data_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variables", ], + tags = [ + "no_oss", # Flaky due to port collisions + ], ) filegroup( diff --git a/tensorflow/contrib/predictor/BUILD b/tensorflow/contrib/predictor/BUILD index 745dc2f8366a319dc94246228a6cc3efc12a53b8..1bf40ab6b26c6ad1f9658a4b0ad93527fe609698 100644 --- a/tensorflow/contrib/predictor/BUILD +++ b/tensorflow/contrib/predictor/BUILD @@ -25,7 +25,10 @@ py_library( srcs = ["__init__.py"], srcs_version = "PY2AND3", visibility = ["//visibility:public"], - deps = [":predictor_factories"], + deps = [ + ":predictor_factories", + "//tensorflow/python:util", + ], ) py_library( @@ -58,7 +61,6 @@ py_library( "//tensorflow/python:session", "//tensorflow/python/saved_model:loader", "//tensorflow/python/saved_model:signature_constants", - "//tensorflow/python/saved_model:signature_def_utils", ], ) diff --git a/tensorflow/contrib/predictor/predictor.py b/tensorflow/contrib/predictor/predictor.py index dbc0028259ebe50bdbe8dee9ef3ccff1aff5507c..28fa815684dd5e242f82d51968d856553315e8d5 100644 --- a/tensorflow/contrib/predictor/predictor.py +++ b/tensorflow/contrib/predictor/predictor.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - """Abstract base class for all predictors.""" from __future__ import absolute_import @@ -66,8 +65,9 @@ class Predictor(object): expected_keys = set(self.feed_tensors.keys()) unexpected_keys = input_keys - expected_keys if unexpected_keys: - raise ValueError('Got unexpected keys in input_dict: {}'.format( - unexpected_keys)) + raise ValueError( + 'Got unexpected keys in input_dict: {}\nexpected: {}'.format( + unexpected_keys, expected_keys)) feed_dict = {} for key in self.feed_tensors.keys(): diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD index 7ff186bc2ad7204d934c322a04ad1c3f2aa383ab..935af80e7a0cb94b9ccdc52b48a73cecc5beb299 100644 --- a/tensorflow/contrib/quantize/BUILD +++ b/tensorflow/contrib/quantize/BUILD @@ -13,6 +13,35 @@ py_library( deps = [], ) +py_library( + name = "graph_matcher", + srcs = [ + "python/graph_matcher.py", + ], + srcs_version = "PY2AND3", + deps = [], +) + +py_test( + name = "graph_matcher_test", + size = "small", + srcs = ["python/graph_matcher_test.py"], + srcs_version = "PY2AND3", + deps = [ + ":graph_matcher", + "//tensorflow/contrib/framework:framework_py", + "//tensorflow/contrib/layers:layers_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:init_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:nn_ops", + "//tensorflow/python:platform_test", + ], +) + py_library( name = "input_to_ops", srcs = ["python/input_to_ops.py"], @@ -43,9 +72,11 @@ py_library( srcs_version = "PY2AND3", deps = [ ":common", + ":graph_matcher", ":input_to_ops", "//tensorflow/contrib/graph_editor:graph_editor_py", "//tensorflow/python:array_ops", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", @@ -147,10 +178,11 @@ py_test( py_test( name = "quantize_parameterized_test", - size = "medium", + size = "large", srcs = ["python/quantize_parameterized_test.py"], srcs_version = "PY2AND3", deps = [ + ":fold_batch_norms", ":quantize", "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:array_ops", @@ -177,6 +209,7 @@ py_library( ":fold_batch_norms", ":quantize", "//tensorflow/python:framework_ops", + "//tensorflow/python:util", "//tensorflow/python:variables", ], ) @@ -188,9 +221,13 @@ py_test( srcs_version = "PY2AND3", deps = [ ":quantize_graph", + "//tensorflow/contrib/layers:layers_py", + "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:init_ops", + "//tensorflow/python:nn_ops", "//tensorflow/python:platform_test", "//tensorflow/python:variables", ], diff --git a/tensorflow/contrib/quantize/python/copy_graph_test.py b/tensorflow/contrib/quantize/python/copy_graph_test.py index 0889f12de6aac53f70ecfa7b70fc19ac7b95a5fe..7ff9ad9f8412d7076bf12d6cf10772244444013f 100644 --- a/tensorflow/contrib/quantize/python/copy_graph_test.py +++ b/tensorflow/contrib/quantize/python/copy_graph_test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Tests for tensorflow.quantized.mangle.copy_graph.""" +"""Tests for copy_graph.""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py index c9d16fb32927855aa14b8b8b33457063e26f6e4d..647d4044001f7be701037d07dc46db86c0aa3a0e 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py @@ -21,7 +21,9 @@ from __future__ import print_function import re from tensorflow.contrib import graph_editor from tensorflow.contrib.quantize.python import common +from tensorflow.contrib.quantize.python import graph_matcher from tensorflow.contrib.quantize.python import input_to_ops +from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn @@ -29,7 +31,270 @@ from tensorflow.python.ops import nn_ops def FoldBatchNorms(graph): - """Finds batch norm layers in the graph, folds them into preceding layers. + """Finds batch norm layers and folds them into preceding layers. + + Folding only affects the following layers: Conv2D, fully connected, depthwise + convolution. + + Args: + graph: Graph to walk and modify. + + Raises: + ValueError: When batch norm folding fails. + """ + _FoldFusedBatchNorms(graph) + _FoldUnfusedBatchNorms(graph) + + +def _FoldFusedBatchNorms(graph): + """Finds fused batch norm layers and folds them into preceding layers. + + Folding only affects the following layers: Conv2D, fully connected, depthwise + convolution. + + Args: + graph: Graph to walk and modify. + + Raises: + ValueError: When batch norm folding fails. + """ + for match in _FindFusedBatchNorms(graph): + scope, sep, _ = match.layer_op.name.rpartition('/') + # Make sure new ops are added to `graph` and put on the same device as + # `bn_op`. The '/' (i.e. `sep`) ensures that we reuse the existing scope + # named `scope`. Otherwise, TF creates a unique scope whose name starts with + # `scope`. + with graph.as_default(), graph.name_scope(scope + sep), ops.device( + match.bn_op.device): + # new weights = old weights * gamma / sqrt(variance + epsilon) + # new biases = -mean * gamma / sqrt(variance + epsilon) + beta + multiplier_tensor = match.gamma_tensor * math_ops.rsqrt( + match.variance_tensor + match.bn_op.get_attr('epsilon')) + bias_tensor = math_ops.subtract( + match.beta_tensor, match.mean_tensor * multiplier_tensor, name='bias') + + # The shape of depthwise weights is different, so we need to reshape the + # multiplier_tensor to ensure that the scaled_weight_tensor has the + # expected shape. + if match.layer_op.type == 'DepthwiseConv2dNative': + new_shape = [ + match.weight_tensor.get_shape().as_list()[2], + match.weight_tensor.get_shape().as_list()[3] + ] + multiplier_tensor = array_ops.reshape( + multiplier_tensor, new_shape, name='scale_reshape') + + # TODO(suharshs): This naming of the following ops needs to carefully + # follow the naming expected by quantize.py. Generalize the quantize code + # to not require these delicate naming conventions. + scaled_weight_tensor = math_ops.multiply( + match.weight_tensor, multiplier_tensor, name='mul_fold') + + new_layer_tensor = _CloneWithNewOperands( + match.layer_op, match.input_tensor, scaled_weight_tensor) + + bias_add_tensor = math_ops.add( + new_layer_tensor, bias_tensor, name='add_fold') + + nodes_modified_count = graph_editor.reroute_ts(bias_add_tensor, + match.output_tensor) + if nodes_modified_count != 1: + raise ValueError( + 'Unexpected inputs to op: %s' % match.output_tensor.name) + + +def _CloneWithNewOperands(layer_op, input_tensor, weight_tensor): + """Clones layer_op with input_tensor and weight_tensor as new inputs.""" + new_layer_name = layer_op.name.split('/')[-1] + '_Fold' + if layer_op.type == 'Conv2D': + return nn_ops.conv2d( + input_tensor, + weight_tensor, + strides=layer_op.get_attr('strides'), + padding=layer_op.get_attr('padding'), + use_cudnn_on_gpu=layer_op.get_attr('use_cudnn_on_gpu'), + data_format=layer_op.get_attr('data_format'), + name=new_layer_name) + elif layer_op.type == 'MatMul': + return math_ops.matmul( + input_tensor, + weight_tensor, + transpose_a=layer_op.get_attr('transpose_a'), + transpose_b=layer_op.get_attr('transpose_b'), + name=new_layer_name) + elif layer_op.type == 'DepthwiseConv2dNative': + return nn.depthwise_conv2d( + input_tensor, + weight_tensor, + strides=layer_op.get_attr('strides'), + padding=layer_op.get_attr('padding'), + name=new_layer_name) + else: + raise ValueError('Cannot handle operation of type: %s' % layer_op.type) + + +def _FindFusedBatchNorms(graph): + """Finds all ops and tensors related to found FusedBatchNorms. + + Args: + graph: Graph to inspect. + + Yields: + _FusedBatchNormMatches. + """ + input_pattern = graph_matcher.OpTypePattern('*') + weight_pattern = graph_matcher.OpTypePattern('*') + gamma_pattern = graph_matcher.OpTypePattern('*') + beta_pattern = graph_matcher.OpTypePattern('*') + mean_pattern = graph_matcher.OpTypePattern('*') + variance_pattern = graph_matcher.OpTypePattern('*') + + conv_pattern = graph_matcher.OpTypePattern( + 'Conv2D|DepthwiseConv2dNative', inputs=[input_pattern, weight_pattern]) + # MatMul has a Reshape between it and FusedBatchNorm. + matmul_pattern = graph_matcher.OpTypePattern( + 'MatMul', inputs=[input_pattern, weight_pattern]) + matmul_reshape_pattern = graph_matcher.OpTypePattern( + 'Reshape', inputs=[matmul_pattern, + graph_matcher.OpTypePattern('*')]) + + conv_batch_norm_pattern = graph_matcher.OpTypePattern( + 'FusedBatchNorm', + inputs=[ + conv_pattern, gamma_pattern, beta_pattern, mean_pattern, + variance_pattern + ]) + matmul_batch_norm_pattern = graph_matcher.OpTypePattern( + 'FusedBatchNorm', + inputs=[ + matmul_reshape_pattern, gamma_pattern, beta_pattern, mean_pattern, + variance_pattern + ]) + matmul_bn_output_reshape_pattern = graph_matcher.OpTypePattern( + 'Reshape', + inputs=[matmul_batch_norm_pattern, + graph_matcher.OpTypePattern('*')]) + + conv_matcher = graph_matcher.GraphMatcher(conv_batch_norm_pattern) + matmul_matcher = graph_matcher.GraphMatcher(matmul_bn_output_reshape_pattern) + + def _GetCommonTensors(match_result): + """Gets tensors needed for FusedBatchNormMatch from match_result.""" + input_tensor = match_result.get_tensor(input_pattern) + weight_tensor = match_result.get_tensor(weight_pattern) + gamma_tensor = match_result.get_tensor(gamma_pattern) + beta_tensor = match_result.get_tensor(beta_pattern) + # FusedBatchNorm in training is different from that in inference. It takes + # empty 'mean' and empty 'variance', and produces the mean and the variance + # of the batch. Therefore, when is_training is true, mean_tensor and + # variance_tensor point to 1st and 2nd (0-based) output of bn_op, + # respectively; when is_training is false, they point to bn_op's inputs. + is_training = bn_op.get_attr('is_training') + if is_training: + mean_tensor = bn_op.outputs[1] + variance_tensor = bn_op.outputs[2] + else: + mean_tensor = match_result.get_tensor(mean_pattern) + variance_tensor = match_result.get_tensor(variance_pattern) + return (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor) + + for match_result in conv_matcher.match_graph(graph): + layer_op = match_result.get_op(conv_pattern) + bn_op = match_result.get_op(conv_batch_norm_pattern) + # In the case of convolution the output_tensor is the output of bn_op. + output_tensor = bn_op.outputs[0] + + (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor) = _GetCommonTensors(match_result) + yield _FusedBatchNormMatch( + layer_op=layer_op, + bn_op=bn_op, + output_tensor=output_tensor, + input_tensor=input_tensor, + weight_tensor=weight_tensor, + gamma_tensor=gamma_tensor, + beta_tensor=beta_tensor, + mean_tensor=mean_tensor, + variance_tensor=variance_tensor) + + for match_result in matmul_matcher.match_graph(graph): + layer_op = match_result.get_op(matmul_pattern) + bn_op = match_result.get_op(matmul_batch_norm_pattern) + # In the MatMul case, the output of batch norm is reshaped back into a + # 2D tensor, so the output_tensor is the output of the Reshape op. + output_reshape_op = match_result.get_op(matmul_bn_output_reshape_pattern) + output_tensor = output_reshape_op.outputs[0] + + (input_tensor, weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor) = _GetCommonTensors(match_result) + yield _FusedBatchNormMatch( + layer_op=layer_op, + bn_op=bn_op, + output_tensor=output_tensor, + input_tensor=input_tensor, + weight_tensor=weight_tensor, + gamma_tensor=gamma_tensor, + beta_tensor=beta_tensor, + mean_tensor=mean_tensor, + variance_tensor=variance_tensor) + + +class _FusedBatchNormMatch(object): + """Contains all information related to a found FusedBatchNorm.""" + + def __init__(self, layer_op, bn_op, output_tensor, input_tensor, + weight_tensor, gamma_tensor, beta_tensor, mean_tensor, + variance_tensor): + self._layer_op = layer_op + self._bn_op = bn_op + self._output_tensor = output_tensor + self._input_tensor = input_tensor + self._weight_tensor = weight_tensor + self._gamma_tensor = gamma_tensor + self._beta_tensor = beta_tensor + self._mean_tensor = mean_tensor + self._variance_tensor = variance_tensor + + @property + def layer_op(self): + return self._layer_op + + @property + def bn_op(self): + return self._bn_op + + @property + def output_tensor(self): + return self._output_tensor + + @property + def input_tensor(self): + return self._input_tensor + + @property + def weight_tensor(self): + return self._weight_tensor + + @property + def gamma_tensor(self): + return self._gamma_tensor + + @property + def beta_tensor(self): + return self._beta_tensor + + @property + def mean_tensor(self): + return self._mean_tensor + + @property + def variance_tensor(self): + return self._variance_tensor + + +def _FoldUnfusedBatchNorms(graph): + """Finds unfused batch norm layers and folds them into preceding layers. Folding only affects the following layers: Conv2D, fully connected, depthwise convolution. diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py index 4f11188a551fa7054bf7c91f70ec9f3f591a4c8e..2cecf6851467f82675bd67bf1fb108e9a39df1b0 100644 --- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py +++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import copy from tensorflow.contrib.layers.python.layers import layers from tensorflow.contrib.quantize.python import fold_batch_norms from tensorflow.python.framework import dtypes @@ -35,29 +34,32 @@ conv2d = layers.conv2d fully_connected = layers.fully_connected separable_conv2d = layers.separable_conv2d -_DEFAULT_BATCH_NORM_PARAMS = { - 'center': True, - 'scale': True, - 'decay': 1.0 - 0.003, - 'fused': False, -} - # TODO(suharshs): Use parameterized test once OSS TF supports it. class FoldBatchNormsTest(test_util.TensorFlowTestCase): def _RunTestOverParameters(self, test_fn): parameters_list = [ - # (relu, relu_op_name, with_bypass) - (nn_ops.relu6, 'Relu6', False), - (nn_ops.relu, 'Relu', False), - (nn_ops.relu6, 'Relu6', True), - (nn_ops.relu, 'Relu', True), + # (relu, relu_op_name, with_bypass, has_scaling, fused_batch_norm) + (nn_ops.relu6, 'Relu6', False, False, False), + (nn_ops.relu, 'Relu', False, False, False), + (nn_ops.relu6, 'Relu6', True, False, False), + (nn_ops.relu, 'Relu', True, False, False), + (nn_ops.relu6, 'Relu6', False, True, False), + (nn_ops.relu, 'Relu', False, True, False), + (nn_ops.relu6, 'Relu6', True, True, False), + (nn_ops.relu, 'Relu', True, True, False), + # Fused batch norm always has scaling enabled. + (nn_ops.relu6, 'Relu6', False, True, True), + (nn_ops.relu, 'Relu', False, True, True), + (nn_ops.relu6, 'Relu6', True, True, True), + (nn_ops.relu, 'Relu', True, True, True), ] - for parameters in parameters_list: - test_fn(parameters[0], parameters[1], parameters[2]) + for params in parameters_list: + test_fn(params[0], params[1], params[2], params[3], params[4]) - def _TestFoldConv2d(self, relu, relu_op_name, with_bypass): + def _TestFoldConv2d(self, relu, relu_op_name, with_bypass, has_scaling, + fused_batch_norm): """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. Args: @@ -65,6 +67,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -74,12 +78,17 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): stride = 1 if with_bypass else 2 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) + node = conv2d( + inputs, + out_depth, [5, 5], + stride=stride, + padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), + scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') relu(node, name='test/' + relu_op_name) @@ -88,12 +97,13 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/mul']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) + self._AssertInputOpsAre(folded_mul, [ + scope + '/weights/read', + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) + ]) + self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) - folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') + folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') self.assertEqual(folded_conv.type, 'Conv2D') self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', inputs.op.name]) @@ -101,16 +111,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/convolution_Fold', - scope + '/BatchNorm/batchnorm/sub']) + self._AssertInputOpsAre(folded_add, [ + scope + '/Conv2D_Fold', + self._BathNormBiasName(scope, fused_batch_norm) + ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) def testFoldConv2d(self): self._RunTestOverParameters(self._TestFoldConv2d) - def _TestFoldConv2dUnknownShape(self, relu, relu_op_name, with_bypass): + def _TestFoldConv2dUnknownShape(self, relu, relu_op_name, with_bypass, + has_scaling, fused_batch_norm): """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. Tests that folding works even with an input shape where some dimensions are @@ -121,6 +133,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -137,7 +151,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): weights_initializer=self._WeightInit(0.09), activation_fn=activation_fn, normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') @@ -148,11 +163,12 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') self._AssertInputOpsAre(folded_mul, [ - scope + '/weights/read', scope + '/BatchNorm/batchnorm/mul' + scope + '/weights/read', + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) ]) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) + self._AssertOutputGoesToOps(folded_mul, g, [scope + '/Conv2D_Fold']) - folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') + folded_conv = g.get_operation_by_name(scope + '/Conv2D_Fold') self.assertEqual(folded_conv.type, 'Conv2D') self._AssertInputOpsAre(folded_conv, [scope + '/mul_fold', inputs.op.name]) self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) @@ -160,7 +176,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') self._AssertInputOpsAre(folded_add, [ - scope + '/convolution_Fold', scope + '/BatchNorm/batchnorm/sub' + scope + '/Conv2D_Fold', + self._BathNormBiasName(scope, fused_batch_norm) ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) @@ -168,62 +185,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): def testFoldConv2dUnknownShape(self): self._RunTestOverParameters(self._TestFoldConv2dUnknownShape) - def _TestFoldConv2dWithoutScale(self, relu, relu_op_name, with_bypass): - """Tests folding cases: inputs -> Conv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - out_depth = 3 if with_bypass else 32 - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS) - bn_params['scale'] = False - scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms(g) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/Rsqrt']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/convolution_Fold']) - - folded_conv = g.get_operation_by_name(scope + '/convolution_Fold') - self.assertEqual(folded_conv.type, 'Conv2D') - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/convolution_Fold', - scope + '/BatchNorm/batchnorm/sub']) - output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - - def testFoldConv2dWithoutScale(self): - self._RunTestOverParameters(self._TestFoldConv2dWithoutScale) - - def _TestFoldFullyConnectedLayer(self, relu, relu_op_name, with_bypass): + def _TestFoldFullyConnectedLayer(self, relu, relu_op_name, with_bypass, + has_scaling, fused_batch_norm): """Tests folding cases: inputs -> FC with batch norm -> Relu*. Args: @@ -231,6 +194,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -239,12 +204,15 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): out_depth = 256 if with_bypass else 128 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' - node = fully_connected(inputs, out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) + node = fully_connected( + inputs, + out_depth, + weights_initializer=self._WeightInit(0.03), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), + scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') relu(node, name='test/' + relu_op_name) @@ -253,9 +221,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_mul = g.get_operation_by_name(scope + '/mul_fold') self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/mul']) + self._AssertInputOpsAre(folded_mul, [ + scope + '/weights/read', + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm) + ]) self._AssertOutputGoesToOps(folded_mul, g, [scope + '/MatMul_Fold']) folded_conv = g.get_operation_by_name(scope + '/MatMul_Fold') @@ -266,71 +235,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/MatMul_Fold', - scope + '/BatchNorm/batchnorm/sub']) + self._AssertInputOpsAre(folded_add, [ + scope + '/MatMul_Fold', + self._BathNormBiasName(scope, fused_batch_norm) + ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) def testFoldFullyConnectedLayer(self): self._RunTestOverParameters(self._TestFoldFullyConnectedLayer) - def _TestFoldFullyConnectedLayerWithoutScale(self, relu, relu_op_name, - with_bypass): - """Tests folding cases: inputs -> FC with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - """ - g = ops.Graph() - with g.as_default(): - batch_size, depth = 5, 256 - inputs = array_ops.zeros((batch_size, depth)) - out_depth = 256 if with_bypass else 128 - activation_fn = None if with_bypass else relu - bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS) - bn_params['scale'] = False - scope = 'test/test2' if with_bypass else 'test' - node = fully_connected(inputs, out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms(g) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/weights/read', - scope + '/BatchNorm/batchnorm/Rsqrt']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/MatMul_Fold']) - - folded_conv = g.get_operation_by_name(scope + '/MatMul_Fold') - self.assertEqual(folded_conv.type, 'MatMul') - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/MatMul_Fold', - scope + '/BatchNorm/batchnorm/sub']) - output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - - def testFoldFullyConnectedLayerWithoutScale(self): - self._RunTestOverParameters(self._TestFoldFullyConnectedLayerWithoutScale) - - def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass): + def _TestFoldDepthwiseConv2d(self, relu, relu_op_name, with_bypass, + has_scaling, fused_batch_norm): """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*. Args: @@ -338,6 +254,8 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): relu_op_name: String, name of the Relu* operation. with_bypass: Bool, when true there is an extra connection added from inputs to just before Relu*. + has_scaling: Bool, when true the batch norm has scaling. + fused_batch_norm: Bool, when true the batch norm is fused. """ g = ops.Graph() with g.as_default(): @@ -346,13 +264,18 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): stride = 1 if with_bypass else 2 activation_fn = None if with_bypass else relu scope = 'test/test2' if with_bypass else 'test' - node = separable_conv2d(inputs, None, [5, 5], stride=stride, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) + node = separable_conv2d( + inputs, + None, [5, 5], + stride=stride, + depth_multiplier=1.0, + padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=activation_fn, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams( + scale=has_scaling, fused=fused_batch_norm), + scope=scope) if with_bypass: node = math_ops.add(inputs, node, name='test/Add') relu(node, name='test/' + relu_op_name) @@ -368,9 +291,10 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): scale_reshape = g.get_operation_by_name(scope + '/scale_reshape') self.assertEqual(scale_reshape.type, 'Reshape') - self._AssertInputOpsAre(scale_reshape, - [scope + '/BatchNorm/batchnorm/mul', - scope + '/scale_reshape/shape']) + self._AssertInputOpsAre(scale_reshape, [ + self._BatchNormMultiplierName(scope, has_scaling, fused_batch_norm), + scope + '/scale_reshape/shape' + ]) self._AssertOutputGoesToOps(scale_reshape, g, [scope + '/mul_fold']) folded_conv = g.get_operation_by_name(scope + '/depthwise_Fold') @@ -381,77 +305,35 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase): folded_add = g.get_operation_by_name(scope + '/add_fold') self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/depthwise_Fold', - scope + '/BatchNorm/batchnorm/sub']) + self._AssertInputOpsAre(folded_add, [ + scope + '/depthwise_Fold', + self._BathNormBiasName(scope, fused_batch_norm) + ]) output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] self._AssertOutputGoesToOps(folded_add, g, output_op_names) def testFoldDepthwiseConv2d(self): self._RunTestOverParameters(self._TestFoldDepthwiseConv2d) - def _TestFoldDepthwiseConv2dWithoutScale(self, relu, relu_op_name, - with_bypass): - """Tests folding: inputs -> DepthwiseConv2d with batch norm -> Relu*. - - Args: - relu: Callable that returns an Operation, a factory method for the Relu*. - relu_op_name: String, name of the Relu* operation. - with_bypass: Bool, when true there is an extra connection added from - inputs to just before Relu*. - """ - g = ops.Graph() - with g.as_default(): - batch_size, height, width = 5, 128, 128 - inputs = array_ops.zeros((batch_size, height, width, 3)) - stride = 1 if with_bypass else 2 - activation_fn = None if with_bypass else relu - bn_params = copy.copy(_DEFAULT_BATCH_NORM_PARAMS) - bn_params['scale'] = False - scope = 'test/test2' if with_bypass else 'test' - node = separable_conv2d(inputs, None, [5, 5], stride=stride, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=activation_fn, - normalizer_fn=batch_norm, - normalizer_params=bn_params, - scope=scope) - if with_bypass: - node = math_ops.add(inputs, node, name='test/Add') - relu(node, name='test/' + relu_op_name) - - fold_batch_norms.FoldBatchNorms(g) - - folded_mul = g.get_operation_by_name(scope + '/mul_fold') - self.assertEqual(folded_mul.type, 'Mul') - self._AssertInputOpsAre(folded_mul, - [scope + '/depthwise_weights/read', - scope + '/scale_reshape']) - self._AssertOutputGoesToOps(folded_mul, g, [scope + '/depthwise_Fold']) - - scale_reshape = g.get_operation_by_name(scope + '/scale_reshape') - self.assertEqual(scale_reshape.type, 'Reshape') - self._AssertInputOpsAre(scale_reshape, - [scope + '/BatchNorm/batchnorm/Rsqrt', - scope + '/scale_reshape/shape']) - self._AssertOutputGoesToOps(scale_reshape, g, [scope + '/mul_fold']) - - folded_conv = g.get_operation_by_name(scope + '/depthwise_Fold') - self.assertEqual(folded_conv.type, 'DepthwiseConv2dNative') - self._AssertInputOpsAre(folded_conv, - [scope + '/mul_fold', inputs.op.name]) - self._AssertOutputGoesToOps(folded_conv, g, [scope + '/add_fold']) - - folded_add = g.get_operation_by_name(scope + '/add_fold') - self.assertEqual(folded_add.type, 'Add') - self._AssertInputOpsAre(folded_add, - [scope + '/depthwise_Fold', - scope + '/BatchNorm/batchnorm/sub']) - output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name] - self._AssertOutputGoesToOps(folded_add, g, output_op_names) - - def testFoldDepthwiseConv2dWithoutScale(self): - self._RunTestOverParameters(self._TestFoldDepthwiseConv2dWithoutScale) + def _BatchNormParams(self, scale=True, fused=False): + return { + 'center': True, + 'scale': scale, + 'decay': 1.0 - 0.003, + 'fused': fused + } + + def _BatchNormMultiplierName(self, scope, has_scaling, fused): + if has_scaling: + if fused: + return scope + '/mul' + return scope + '/BatchNorm/batchnorm/mul' + return scope + '/BatchNorm/batchnorm/Rsqrt' + + def _BathNormBiasName(self, scope, fused): + if fused: + return scope + '/bias' + return scope + '/BatchNorm/batchnorm/sub' def _WeightInit(self, stddev): """Returns a truncated normal variable initializer. diff --git a/tensorflow/contrib/quantize/python/graph_matcher.py b/tensorflow/contrib/quantize/python/graph_matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..e3581cc55905a0af7d0464bc0ec673d3ed7f0363 --- /dev/null +++ b/tensorflow/contrib/quantize/python/graph_matcher.py @@ -0,0 +1,200 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities that match patterns in a tf.Graph.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +class OpTypePattern(object): + """A tree pattern that matches TF expressions with certain op types.""" + + def __init__(self, op_type, name=None, inputs=None): + """Initializes an OpTypePattern. + + Args: + op_type: string that specifies the allowed types of the root. It can be + (1) an op type, e.g. 'Conv2D', + (2) '*', i.e. wildcard, or + (3) multiple op types separated by '|', e.g., 'Relu|Relu6'. + We could use regex strings, which might be worthwhile when we have many + similar TF op types. + name: Optional string. The name of the pattern that can be looked up in + MatchResult. + inputs: Optional list of `OpTypePattern`s or strings that specify the + patterns for the inputs of a matching op. If None, this pattern accepts + any inputs of a matching op. + """ + self._op_type = op_type + self._name = name + if inputs is None: + inputs = [] + self._inputs = [ + input_pattern if isinstance(input_pattern, OpTypePattern) else + OpTypePattern(input_pattern) for input_pattern in inputs + ] + + @property + def op_type(self): + return self._op_type + + @property + def inputs(self): + return self._inputs + + @property + def name(self): + return self._name + + +class MatchResult(object): + r"""Encapsulates the result of a match done by GraphMatcher. + + MatchResult contains a map from OpTypePattern to the matching op and tensor. + When the matching op has multiple output tensors, the matching tensor is the + output tensor used by the matching op of the parent pattern. E.g., when we + match graph + + - + + / \y0 y1/ \ + x split z + | + y (nodes are ops; edges are going up) + + against add_pattern defined as + + y1_pattern = OpTypePattern('*') + z_pattern = OpTypePattern('*') + add_pattern = OpTypePattern('+', inputs=[y1_pattern, z_pattern]) + + the matching op of `y1_pattern` is `split`, and the matching tensor of + `y1_pattern` + is `y1` not `y0`. + """ + + def __init__(self): + self._pattern_to_op_tensor = {} + self._name_to_pattern = {} + + def add(self, pattern, op, tensor): + self._pattern_to_op_tensor[pattern] = op, tensor + if pattern.name is not None: + if pattern.name in self._name_to_pattern: + raise ValueError( + 'Name %s is already bound to another pattern' % pattern.name) + self._name_to_pattern[pattern.name] = pattern + + def _to_pattern(self, pattern_or_name): + if isinstance(pattern_or_name, OpTypePattern): + return pattern_or_name + + if isinstance(pattern_or_name, str): + return self._name_to_pattern[pattern_or_name] + + raise ValueError('pattern_or_name has type %s. Expect OpTypePattern or str.' + % type(pattern_or_name)) + + def get_op(self, pattern_or_name): + return self._pattern_to_op_tensor[self._to_pattern(pattern_or_name)][0] + + def get_tensor(self, pattern_or_name): + return self._pattern_to_op_tensor[self._to_pattern(pattern_or_name)][1] + + +class GraphMatcher(object): + """Checks if a particular subgraph matches a given pattern.""" + + def __init__(self, pattern): + """Initializes a GraphMatcher. + + Args: + pattern: The `OpTypePattern` against which `GraphMatcher` matches + subgraphs. + """ + self._pattern = pattern + + def _match_pattern(self, pattern, op, tensor): + """Returns whether an TF expression rooted at `op` matches `pattern`. + + If there is a match, adds to `self._match_result` the matching op and tensor + with key `pattern`. + + Args: + pattern: An `OpTypePattern`. + op: A `tf.Operation` to match against the pattern. + tensor: the output `tf.Tensor` of `op` that is used by the matching op of + `pattern`'s parent. Can be None if `pattern` is already the root of the + pattern tree. + + Returns: + True if an TF expression rooted at `op` matches `pattern`. + """ + if pattern.op_type != '*': + if op.type not in pattern.op_type.split('|'): + return False + + self._match_result.add(pattern, op, tensor) + + if not pattern.inputs: + # If pattern.inputs is empty, skips the rest and accepts all the inputs. + return True + + return len(op.inputs) == len(pattern.inputs) and all([ + self._match_pattern(input_pattern, input_tensor.op, input_tensor) + for input_tensor, input_pattern in zip(op.inputs, pattern.inputs) + ]) + + def match_op(self, op): + """Matches `op` against `self._pattern`. + + Args: + op: `tf.Operation` to match against the pattern. + + Returns: + Returns a `MatchResult` if `op` matches the pattern; otherwise, returns + None. + """ + self._match_result = MatchResult() + if not self._match_pattern(self._pattern, op, tensor=None): + return None + return self._match_result + + def match_ops(self, ops): + """Matches each operation in `ops` against `self._pattern`. + + Args: + ops: collection of `tf.Operation` to match against the pattern. + + Yields: + `MatchResult` for each `tf.Operation` that matches the pattern. + """ + for op in ops: + match_result = self.match_op(op) + if match_result: + yield match_result + + def match_graph(self, graph): + """Matches each operation in `graph` against `self._pattern`. + + Args: + graph: `tf.Graph` containing operations to match. + + Yields: + `MatchResult` for each `tf.Operation` in `graph` that matches the pattern. + """ + # Python 3.3.2+ implements `yield from`, but for now: + for match_result in self.match_ops(graph.get_operations()): + yield match_result diff --git a/tensorflow/contrib/quantize/python/graph_matcher_test.py b/tensorflow/contrib/quantize/python/graph_matcher_test.py new file mode 100644 index 0000000000000000000000000000000000000000..e1572865e423e569ee3b280036c0e02b71b70648 --- /dev/null +++ b/tensorflow/contrib/quantize/python/graph_matcher_test.py @@ -0,0 +1,130 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for graph_matcher.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.framework.python import ops as contrib_ops +from tensorflow.contrib.layers.python.layers import initializers +from tensorflow.contrib.layers.python.layers import layers +from tensorflow.contrib.quantize.python import graph_matcher +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import googletest + + +class GraphMatcherTest(test_util.TensorFlowTestCase): + + def test_conv_layer(self): + g = ops.Graph() + with g.as_default(): + inputs = array_ops.placeholder(dtypes.float32, shape=[8, 5, 5, 3]) + + with contrib_ops.arg_scope( + [layers.batch_norm], fused=True, is_training=True, trainable=True): + return layers.convolution( + inputs, + num_outputs=16, + kernel_size=3, + stride=1, + padding='VALID', + activation_fn=nn_ops.relu, + normalizer_fn=layers.batch_norm, + normalizer_params={}, + weights_initializer=initializers.xavier_initializer(), + weights_regularizer=None, + biases_initializer=init_ops.zeros_initializer(), + biases_regularizer=None, + reuse=None, + trainable=True, + scope=None) + + inputs_pattern = graph_matcher.OpTypePattern('*', name='inputs') + relu_pattern = graph_matcher.OpTypePattern( + 'Relu', + name='relu', + inputs=[ + graph_matcher.OpTypePattern( + 'FusedBatchNorm', + inputs=[ + graph_matcher.OpTypePattern( + 'Conv2D', inputs=[inputs_pattern, '*']), '*', '*', '*', + '*' + ]) + ]) + matcher = graph_matcher.GraphMatcher(relu_pattern) + match_results = list(matcher.match_graph(g)) + self.assertEqual(1, len(match_results)) + match_result = match_results[0] + self.assertEqual(match_result.get_tensor(inputs_pattern), inputs) + self.assertEqual(match_result.get_tensor('inputs'), inputs) + + def test_multiple_outputs(self): + # - + + # / \y0 y1/ \ + # x split z + # | + # y (nodes are ops; edges are going up) + g = ops.Graph() + with g.as_default(): + x = array_ops.placeholder(dtypes.float32, shape=[1], name='x') + y = array_ops.placeholder(dtypes.float32, shape=[2], name='y') + y0, y1 = array_ops.split(y, num_or_size_splits=2, axis=0) + z = array_ops.placeholder(dtypes.float32, shape=[1], name='z') + math_ops.add(x, y0) + math_ops.subtract(y1, z) + + y1_pattern = graph_matcher.OpTypePattern('*') + minus_pattern = graph_matcher.OpTypePattern('Sub', inputs=[y1_pattern, '*']) + matcher = graph_matcher.GraphMatcher(minus_pattern) + + match_results = list(matcher.match_graph(g)) + self.assertEqual(1, len(match_results)) + match_result = match_results[0] + + self.assertEqual(y0.op, y1.op) + self.assertEqual(match_result.get_op(y1_pattern), y1.op) + self.assertEqual(match_result.get_tensor(y1_pattern), y1) + + def test_oneof_pattern(self): + # - + + # / \ / \ + # x y z + g = ops.Graph() + with g.as_default(): + x = array_ops.placeholder(dtypes.float32, shape=[], name='x') + y = array_ops.placeholder(dtypes.float32, shape=[], name='y') + z = array_ops.placeholder(dtypes.float32, shape=[], name='z') + plus = x + y + minus = y - z + + add_or_sub_pattern = graph_matcher.OpTypePattern( + 'Add|Sub', inputs=['*', '*']) + matcher = graph_matcher.GraphMatcher(add_or_sub_pattern) + self.assertEqual([ + match_result.get_op(add_or_sub_pattern) + for match_result in matcher.match_graph(g) + ], [plus.op, minus.op]) + + +if __name__ == '__main__': + googletest.main() diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py index aaf3e92b8ea518fbbe55628b856e0191c949c619..d647bb94e849c713c2aca93c53f372bae5857c43 100644 --- a/tensorflow/contrib/quantize/python/quantize_graph.py +++ b/tensorflow/contrib/quantize/python/quantize_graph.py @@ -25,7 +25,10 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import variables -def _create_graph(input_graph, is_training, elements=None): +def _create_graph(input_graph, + is_training, + elements=None, + device_name_or_function=None): """Returns a transformed training input_graph for simulated quantization. The forward pass has fake quantization ops inserted to simulate the error @@ -36,12 +39,12 @@ def _create_graph(input_graph, is_training, elements=None): is_training: Whether quantizing training or eval graph. elements: (Optional) List of Tensors and Operations in input_graph whose corresponding elements in the new graph will be returned. + device_name_or_function: (Optional) The device name or function to use. Returns: - Returns a tuple(g, l) where: g is new tf.Graph that is rewritten for simulated quantization. l is a list of Tensors/Operations in g corresponding to the provided input - elements. + elements, if elements is not None. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or @@ -49,11 +52,14 @@ def _create_graph(input_graph, is_training, elements=None): """ # TODO(suharshs): Describe the process in more detail in the doc string. g = copy_graph.CopyGraph(input_graph) - fold_batch_norms.FoldBatchNorms(g) - quantize.Quantize(g, is_training=is_training) - return_elements = [] + with g.as_default(): + with ops.device(device_name_or_function): + fold_batch_norms.FoldBatchNorms(g) + quantize.Quantize(g, is_training=is_training) if elements is None: - elements = [] + return g + + return_elements = [] for element in elements: if isinstance(element, (ops.Tensor, variables.Variable)): return_elements.append(g.get_tensor_by_name(element.name)) @@ -66,7 +72,9 @@ def _create_graph(input_graph, is_training, elements=None): return g, return_elements -def create_training_graph(input_graph, elements=None): +def create_training_graph(input_graph, + elements=None, + device_name_or_function=None): """Returns a transformed training input_graph for simulated quantization. The forward pass has fake quantization ops inserted to simulate the error @@ -76,21 +84,25 @@ def create_training_graph(input_graph, elements=None): input_graph: The tf.Graph to be transformed. elements: (Optional) List of Tensors and Operations in input_graph whose corresponding elements in the new graph will be returned. + device_name_or_function: (Optional) The device name or function to use. Returns: - Returns a tuple(g, l) where: g is new tf.Graph that is rewritten for simulated quantization. l is a list of Tensors/Operations in g corresponding to the provided input - elements. + elements, if elements is not None. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or tf.Operation. """ - return _create_graph(input_graph, True, elements) + return _create_graph( + input_graph=input_graph, + is_training=True, + elements=elements, + device_name_or_function=device_name_or_function) -def create_eval_graph(input_graph, elements=None): +def create_eval_graph(input_graph, elements=None, device_name_or_function=None): """Returns a transformed eval input_graph for simulated quantization. The forward pass has fake quantization ops inserted to simulate the error @@ -100,15 +112,19 @@ def create_eval_graph(input_graph, elements=None): input_graph: The tf.Graph to be transformed. elements: (Optional) List of Tensors and Operations in input_graph whose corresponding elements in the new graph will be returned. + device_name_or_function: (Optional) The device name or function to use. Returns: - Returns a tuple(g, l) where: g is new tf.Graph that is rewritten for simulated quantization. l is a list of Tensors/Operations in g corresponding to the provided input - elements. + elements, if elements is not None. Raises: ValueError: If elements contains an element that isn't a tf.Tensor or tf.Operation. """ - return _create_graph(input_graph, False, elements) + return _create_graph( + input_graph=input_graph, + is_training=False, + elements=elements, + device_name_or_function=device_name_or_function) diff --git a/tensorflow/contrib/quantize/python/quantize_graph_test.py b/tensorflow/contrib/quantize/python/quantize_graph_test.py index 382076672a70c873ae7c1384e0706231a0ba8a55..3407ace3914fe2de2506a2952ea5d1bf19028bb9 100644 --- a/tensorflow/contrib/quantize/python/quantize_graph_test.py +++ b/tensorflow/contrib/quantize/python/quantize_graph_test.py @@ -18,29 +18,41 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.contrib.layers.python.layers import layers from tensorflow.contrib.quantize.python import quantize_graph from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import nn_ops from tensorflow.python.ops import variables from tensorflow.python.platform import googletest -class QuantizeTest(test_util.TensorFlowTestCase): +class QuantizeGraphTest(test_util.TensorFlowTestCase): # We have a lot of other tests that test the details of the rewrite, here we # just the specific features of the quantize_graph API. def testReturnedElementsTraining(self): + self._TestReturnElements(True) + + def testReturnedElementsEval(self): + self._TestReturnElements(False) + + def _TestReturnElements(self, is_training): graph = ops.Graph() with graph.as_default(): a = constant_op.constant(1.0) b = variables.Variable(2.0) c = a + b elements = [a, b, c.op] - for element in elements: - print(element) - q_graph, returned_elements = quantize_graph.create_training_graph( - graph, elements=elements) + if is_training: + q_graph, returned_elements = quantize_graph.create_training_graph( + graph, elements=elements) + else: + q_graph, returned_elements = quantize_graph.create_eval_graph( + graph, elements=elements) # Make sure q_graph is different from graph. self.assertTrue(graph != q_graph) # Check that the returned elements are part of the new graph. @@ -50,25 +62,79 @@ class QuantizeTest(test_util.TensorFlowTestCase): for element, returned_element in zip(elements, returned_elements): self.assertEqual(element.name, returned_element.name) - # We have a lot of other tests that test the details of the rewrite, here we - # just the specific features of the quantize_graph API. - def testReturnedElementsEval(self): + def testNoReturnElementsTraining(self): + self._TestNoReturnElements(True) + + def testNoReturnElementsEval(self): + self._TestNoReturnElements(False) + + def _TestNoReturnElements(self, is_training): graph = ops.Graph() with graph.as_default(): a = constant_op.constant(1.0) b = variables.Variable(2.0) - c = a + b - elements = [a, b, c.op] - q_graph, returned_elements = quantize_graph.create_eval_graph( - graph, elements=elements) + _ = a + b + if is_training: + q_graph = quantize_graph.create_training_graph(graph) + else: + q_graph = quantize_graph.create_eval_graph(graph) + # Check that quantize_graph didn't return a tuple when elements isn't + # provided. + self.assertTrue(isinstance(q_graph, ops.Graph)) # Make sure q_graph is different from graph. self.assertTrue(graph != q_graph) - # Check that the returned elements are part of the new graph. - for returned_element in returned_elements: - self.assertEqual(q_graph, returned_element.graph) - # Check that the elements match with the one from the input graph. - for element, returned_element in zip(elements, returned_elements): - self.assertEqual(element.name, returned_element.name) + + def testDeviceNameTraining(self): + self._TestDeviceName(True) + + def testDeviceNameEval(self): + self._TestDeviceName(False) + + def _TestDeviceName(self, is_training): + graph = ops.Graph() + with graph.as_default(): + batch_size, height, width, depth = 5, 128, 128, 3 + inputs = array_ops.zeros((batch_size, height, width, depth)) + conv = layers.conv2d( + inputs, + 32, [5, 5], + stride=2, + padding='SAME', + weights_initializer=self._WeightInit(0.09), + activation_fn=None, + scope='test') + _ = nn_ops.relu6(conv) + + device_name = '/job:oink/task:0/device:CPU:0' + if is_training: + q_graph = quantize_graph.create_training_graph( + graph, device_name_or_function=device_name) + else: + q_graph = quantize_graph.create_eval_graph( + graph, device_name_or_function=device_name) + + orig_variable_names = set( + [v.name for v in graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)]) + q_variables = q_graph.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + # Ensure that variables were added. + self.assertTrue(len(orig_variable_names) < len(q_variables)) + # All added variables should have the specified device name. + for var in q_variables: + if var.name not in orig_variable_names: + self.assertEqual(var.device, device_name) + + def _WeightInit(self, stddev): + """Returns truncated normal variable initializer. + + Function is defined purely to shorten the name so that it stops wrapping. + + Args: + stddev: Standard deviation of normal variable. + + Returns: + An initialized that initialzes with a truncated normal variable. + """ + return init_ops.truncated_normal_initializer(stddev=stddev) if __name__ == '__main__': diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py index b5a32a7266a4c3ddf9a481fd9b292ab0f1812a9a..3e62f95bd63db3134ba0b96c46b4a92aa73ebef9 100644 --- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py +++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.contrib.layers.python.layers import layers +from tensorflow.contrib.quantize.python import fold_batch_norms from tensorflow.contrib.quantize.python import quantize from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -35,18 +36,11 @@ conv2d = layers.conv2d fully_connected = layers.fully_connected separable_conv2d = layers.separable_conv2d -_DEFAULT_BATCH_NORM_PARAMS = { - 'center': True, - 'scale': True, - 'decay': 1.0 - 0.003, - 'fused': False, -} - -# TODO(suharshs): Use parameterized test once OSS TF supports it. class QuantizeTest(test_util.TensorFlowTestCase): - def _RunTestOverParameters(self, test_fn): + def _RunWithoutBatchNormTestOverParameters(self, test_fn): + # TODO(suharshs): Use parameterized test once OSS TF supports it. parameters_list = [ # (activation, activation_op_name, with_bypass, delay) (nn_ops.relu6, 'Relu6', False, None), @@ -60,10 +54,10 @@ class QuantizeTest(test_util.TensorFlowTestCase): (array_ops.identity, 'Identity', True, None), (nn_ops.relu6, 'Relu6', True, 5000), (nn_ops.relu, 'Relu', True, 5000), - (array_ops.identity, 'Identity', True, 5000) + (array_ops.identity, 'Identity', True, 5000), ] - for parameters in parameters_list: - test_fn(parameters[0], parameters[1], parameters[2], parameters[3]) + for params in parameters_list: + test_fn(params[0], params[1], params[2], params[3]) def _TestQuantize_Conv2dWithoutBatchNorm(self, activation, activation_op_name, with_bypass, delay): @@ -107,7 +101,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): scope + '/weights/read' ] self._AssertInputOpsAre(weights_quant, expected_inputs) - output_op_name = scope + '/convolution' + output_op_name = scope + '/Conv2D' self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name]) if with_bypass: @@ -137,7 +131,8 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def testQuantize_Conv2dWithoutBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_Conv2dWithoutBatchNorm) + self._RunWithoutBatchNormTestOverParameters( + self._TestQuantize_Conv2dWithoutBatchNorm) def _TestQuantize_FCWithoutBatchNorm(self, activation, activation_op_name, with_bypass, delay): @@ -210,7 +205,8 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def testQuantize_FCWithoutBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_FCWithoutBatchNorm) + self._RunWithoutBatchNormTestOverParameters( + self._TestQuantize_FCWithoutBatchNorm) def _TestQuantize_DepthwiseConv2dWithoutBatchNorm( self, activation, activation_op_name, with_bypass, delay): @@ -284,11 +280,43 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def testQuantize_DepthwiseConv2dWithoutBatchNorm(self): - self._RunTestOverParameters( + self._RunWithoutBatchNormTestOverParameters( self._TestQuantize_DepthwiseConv2dWithoutBatchNorm) + def _RunBatchNormTestOverParameters(self, test_fn): + # TODO(suharshs): Use parameterized test once OSS TF supports it. + parameters_list = [ + # (activation, activation_op_name, with_bypass, delay, fused_batch_norm) + (nn_ops.relu6, 'Relu6', False, None, False), + (nn_ops.relu, 'Relu', False, None, False), + (array_ops.identity, 'Identity', False, None, False), + (nn_ops.relu6, 'Relu6', False, 5000, False), + (nn_ops.relu, 'Relu', False, 5000, False), + (array_ops.identity, 'Identity', False, 5000, False), + (nn_ops.relu6, 'Relu6', True, None, False), + (nn_ops.relu, 'Relu', True, None, False), + (array_ops.identity, 'Identity', True, None, False), + (nn_ops.relu6, 'Relu6', True, 5000, False), + (nn_ops.relu, 'Relu', True, 5000, False), + (array_ops.identity, 'Identity', True, 5000, False), + (nn_ops.relu6, 'Relu6', False, None, True), + (nn_ops.relu, 'Relu', False, None, True), + (array_ops.identity, 'Identity', False, None, True), + (nn_ops.relu6, 'Relu6', False, 5000, True), + (nn_ops.relu, 'Relu', False, 5000, True), + (array_ops.identity, 'Identity', False, 5000, True), + (nn_ops.relu6, 'Relu6', True, None, True), + (nn_ops.relu, 'Relu', True, None, True), + (array_ops.identity, 'Identity', True, None, True), + (nn_ops.relu6, 'Relu6', True, 5000, True), + (nn_ops.relu, 'Relu', True, 5000, True), + (array_ops.identity, 'Identity', True, 5000, True) + ] + for params in parameters_list: + test_fn(params[0], params[1], params[2], params[3], params[4]) + def _TestQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay): + with_bypass, delay, fused_batch_norm): """Tests quantization: inputs -> Conv2d with batch norm -> Activation. Args: @@ -298,25 +326,29 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. """ self._testQuantize_Conv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=True) self._testQuantize_Conv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=False) def testQuantize_Conv2dWithBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_Conv2dWithBatchNorm) + self._RunBatchNormTestOverParameters(self._TestQuantize_Conv2dWithBatchNorm) def _testQuantize_Conv2dWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay, use_ema): + with_bypass, delay, fused_batch_norm, + use_ema): """Tests quantization: inputs -> Conv2d with batch norm -> Activation. Args: @@ -326,6 +358,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. use_ema: Bool, when true uses EMA quantization for BN folded weights. """ graph = ops.Graph() @@ -337,39 +370,29 @@ class QuantizeTest(test_util.TensorFlowTestCase): stride = 1 if with_bypass else 2 out_depth = 3 if with_bypass else 32 scope = 'test/test2' if with_bypass else 'test' - node = conv2d(inputs, out_depth, [5, 5], stride=stride, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) - # Manually fold the batch norm. - weights = graph.get_operation_by_name(scope + '/weights/read').outputs[0] - bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul') - .outputs[0]) - mul_fold = math_ops.multiply(weights, bn_mult, name=scope + '/mul_fold') - stride = [stride, stride] - conv_fold = nn_ops.convolution( - input=inputs, - filter=mul_fold, + node = conv2d( + inputs, + out_depth, [5, 5], + stride=stride, padding='SAME', - strides=stride, - data_format='NHWC', - name=scope + '/convolution_Fold') - bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub') - .outputs[0]) - add_fold = math_ops.add(conv_fold, bn_bias, name=scope + '/add_fold') + weights_initializer=self._WeightInit(0.09), + activation_fn=None, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams(fused_batch_norm), + scope=scope) + # Manually add a bypass (optionaly) and an activation. if with_bypass: - node = math_ops.add(inputs, add_fold, name='test/Add') - else: - node = add_fold + node = math_ops.add(inputs, node, name='test/Add') + node = activation(node, name='test/' + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') + fold_batch_norms.FoldBatchNorms(graph) + quantize.Quantize( graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema) @@ -384,7 +407,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): ] self._AssertInputOpsAre(weights_quant, expected_inputs) output_op_name = scope + ('/weights_quant/delayed_quant/Switch_1' - if (delay and use_ema) else '/convolution_Fold') + if (delay and use_ema) else '/Conv2D_Fold') self._AssertOutputGoesToOps(weights_quant, graph, [output_op_name]) if with_bypass: @@ -413,7 +436,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def _TestQuantize_FCWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay): + with_bypass, delay, fused_batch_norm): """Tests quantization: inputs -> FC with batch norm -> Activation. Args: @@ -423,25 +446,29 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. """ self._testQuantize_FCWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=True) self._testQuantize_FCWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=False) def testQuantize_FCWithBatchNorm(self): - self._RunTestOverParameters(self._TestQuantize_FCWithBatchNorm) + self._RunBatchNormTestOverParameters(self._TestQuantize_FCWithBatchNorm) def _testQuantize_FCWithBatchNorm(self, activation, activation_op_name, - with_bypass, delay, use_ema): + with_bypass, delay, fused_batch_norm, + use_ema): """Tests quantization: inputs -> FC with batch norm -> Activation. Args: @@ -451,6 +478,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. use_ema: Bool, when true uses EMA quantization for BN folded weights. """ graph = ops.Graph() @@ -461,32 +489,27 @@ class QuantizeTest(test_util.TensorFlowTestCase): inputs = array_ops.zeros((batch_size, depth)) out_depth = 256 if with_bypass else 128 scope = 'test/test2' if with_bypass else 'test' - node = fully_connected(inputs, out_depth, - weights_initializer=self._WeightInit(0.03), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) - # Manually fold the batch norm. - weights = graph.get_operation_by_name(scope + '/weights/read').outputs[0] - bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul') - .outputs[0]) - mul_fold = math_ops.multiply(weights, bn_mult, name=scope + '/mul_fold') - fc_fold = math_ops.matmul(inputs, mul_fold, name=scope + '/MatMul_Fold') - bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub') - .outputs[0]) - add_fold = math_ops.add(fc_fold, bn_bias, name=scope + '/add_fold') + node = fully_connected( + inputs, + out_depth, + weights_initializer=self._WeightInit(0.03), + activation_fn=None, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams(fused_batch_norm), + scope=scope) + # Manually add a bypass (optionaly) and an activation. if with_bypass: - node = math_ops.add(inputs, add_fold, name='test/Add') - else: - node = add_fold + node = math_ops.add(inputs, node, name='test/Add') + node = activation(node, name='test/' + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') + fold_batch_norms.FoldBatchNorms(graph) + quantize.Quantize( graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema) @@ -530,7 +553,8 @@ class QuantizeTest(test_util.TensorFlowTestCase): self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) def _TestQuantize_DepthwiseConv2dWithBatchNorm( - self, activation, activation_op_name, with_bypass, delay): + self, activation, activation_op_name, with_bypass, delay, + fused_batch_norm): """Tests quantization: inputs -> DWConv2d with batch norm -> Activation. Args: @@ -540,26 +564,30 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. """ self._testQuantize_DepthwiseConv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=True) self._testQuantize_DepthwiseConv2dWithBatchNorm( activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema=False) def testQuantize_DepthwiseConv2dWithBatchNorm(self): - self._RunTestOverParameters( - self._TestQuantize_DepthwiseConv2dWithoutBatchNorm) + self._RunBatchNormTestOverParameters( + self._TestQuantize_DepthwiseConv2dWithBatchNorm) def _testQuantize_DepthwiseConv2dWithBatchNorm( - self, activation, activation_op_name, with_bypass, delay, use_ema): + self, activation, activation_op_name, with_bypass, delay, + fused_batch_norm, use_ema): """Tests quantization: inputs -> DWConv2d with batch norm -> Activation. Args: @@ -569,6 +597,7 @@ class QuantizeTest(test_util.TensorFlowTestCase): with_bypass: Bool, when true there is an extra connection added from inputs to just before Activation. delay: Int (optional), delay in number of steps until quantization starts. + fused_batch_norm: Bool, when true use FusedBatchNorm. use_ema: Bool, when true uses EMA quantization for BN folded weights. """ graph = ops.Graph() @@ -579,46 +608,30 @@ class QuantizeTest(test_util.TensorFlowTestCase): inputs = array_ops.zeros((batch_size, height, width, depth)) stride = 1 if with_bypass else 2 scope = 'test/test2' if with_bypass else 'test' - node = separable_conv2d(inputs, None, [5, 5], stride=stride, - depth_multiplier=1.0, padding='SAME', - weights_initializer=self._WeightInit(0.09), - activation_fn=None, - normalizer_fn=batch_norm, - normalizer_params=_DEFAULT_BATCH_NORM_PARAMS, - scope=scope) - # Manually fold the batch norm. - weights = (graph.get_operation_by_name(scope + '/depthwise_weights/read') - .outputs[0]) - bn_mult = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/mul') - .outputs[0]) - new_shape = [ - weights.get_shape().as_list()[2], weights.get_shape().as_list()[3] - ] - bn_mult_reshaped = array_ops.reshape( - bn_mult, new_shape, name=scope + '/gamma_reshape') - mul_fold = math_ops.multiply( - weights, bn_mult_reshaped, name=scope + '/mul_fold') - stride = [1, stride, stride, 1] - conv_fold = nn_ops.depthwise_conv2d( - input=inputs, - filter=mul_fold, + node = separable_conv2d( + inputs, + None, [5, 5], + stride=stride, + depth_multiplier=1.0, padding='SAME', - strides=stride, - name=scope + '/depthwise_Fold') - bn_bias = (graph.get_operation_by_name(scope + '/BatchNorm/batchnorm/sub') - .outputs[0]) - add_fold = math_ops.add(conv_fold, bn_bias, name=scope + '/add_fold') + weights_initializer=self._WeightInit(0.09), + activation_fn=None, + normalizer_fn=batch_norm, + normalizer_params=self._BatchNormParams(fused_batch_norm), + scope=scope) + # Manually add a bypass (optionaly) and an activation. if with_bypass: - node = math_ops.add(inputs, add_fold, name='test/Add') - else: - node = add_fold + node = math_ops.add(inputs, node, name='test/Add') + node = activation(node, name='test/' + activation_op_name) update_barrier = control_flow_ops.no_op(name='update_barrier') with ops.control_dependencies([update_barrier]): array_ops.identity(node, name='control_dependency') + fold_batch_norms.FoldBatchNorms(graph) + quantize.Quantize( graph, quant_delay=delay, quantize_folded_weights_use_ema=use_ema) quantization_node_name = 'FakeQuantWithMinMaxVars' @@ -660,6 +673,9 @@ class QuantizeTest(test_util.TensorFlowTestCase): if delay else 'control_dependency') self._AssertOutputGoesToOps(act_quant, graph, [output_op_name]) + def _BatchNormParams(self, fused=False): + return {'center': True, 'scale': True, 'decay': 1.0 - 0.003, 'fused': fused} + def _WeightInit(self, stddev): """Returns truncated normal variable initializer. diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py index a6bd809bb7de0b674671d09e4a941675976ce8ab..4a82eac1978cf834732e339e4e76a4507b9a090c 100644 --- a/tensorflow/contrib/quantize/python/quantize_test.py +++ b/tensorflow/contrib/quantize/python/quantize_test.py @@ -65,28 +65,5 @@ class QuantizeTest(test_util.TensorFlowTestCase): """ return init_ops.truncated_normal_initializer(stddev=stddev) - def _AssertInputOpsAre(self, op, in_op_names): - """Asserts that all inputs to op come from in_op_names (disregarding order). - - Args: - op: Operation to check inputs for. - in_op_names: List of strings, operations where all op's inputs should - come from. - """ - expected_inputs = [in_op_name + ':0' for in_op_name in in_op_names] - self.assertItemsEqual([t.name for t in op.inputs], expected_inputs) - - def _AssertOutputGoesToOps(self, op, graph, out_op_names): - """Asserts that outputs from op go to out_op_names (and perhaps others). - - Args: - op: Operation to check outputs for. - graph: Graph where output operations are located. - out_op_names: List of strings, operations where op's outputs should go. - """ - for out_op_name in out_op_names: - out_op = graph.get_operation_by_name(out_op_name) - self.assertIn(op.outputs[0].name, [str(t.name) for t in out_op.inputs]) - if __name__ == '__main__': googletest.main() diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD index ed2f3af08cbbd8ae5da2a87f4a7dd9854493c346..d16b2908a0285e04ef5d3ede2050bf24c508228d 100644 --- a/tensorflow/contrib/receptive_field/BUILD +++ b/tensorflow/contrib/receptive_field/BUILD @@ -39,7 +39,9 @@ py_library( deps = [ ":graph_compute_order_py", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:framework_ops", "//tensorflow/python:platform", + "//third_party/py/numpy", ], ) @@ -49,12 +51,13 @@ py_test( srcs_version = "PY2AND3", deps = [ ":receptive_field_py", - "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/slim", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", + "//tensorflow/python:framework_ops", "//tensorflow/python:nn", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/reduce_slice_ops/BUILD b/tensorflow/contrib/reduce_slice_ops/BUILD index fded03090ea48ecea464d64ac87206700b6476c9..b31f4488f5882a0bc4e419668dba5da72d69b7fe 100644 --- a/tensorflow/contrib/reduce_slice_ops/BUILD +++ b/tensorflow/contrib/reduce_slice_ops/BUILD @@ -71,6 +71,7 @@ tf_custom_op_py_library( ":reduce_slice_ops", "//tensorflow/contrib/util:util_py", "//tensorflow/python:framework", + "//tensorflow/python:platform", ], ) diff --git a/tensorflow/contrib/resampler/BUILD b/tensorflow/contrib/resampler/BUILD index 1b9efd1ecd7d4807fe04b52f2f4148e95fce9a8c..f0ecc8b85a5db93075d3cf0b55e7df95732bcf94 100644 --- a/tensorflow/contrib/resampler/BUILD +++ b/tensorflow/contrib/resampler/BUILD @@ -26,9 +26,15 @@ tf_custom_op_py_library( deps = [ ":resampler_ops", "//tensorflow/contrib/util:util_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:constant_op", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", "//tensorflow/python:framework_ops", "//tensorflow/python:platform", "//tensorflow/python:util", + "//third_party/py/numpy", ], ) diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD index 6395cd8316551336ead99a13594ad1919341c9cd..b70a5bbcd107b4c21e09c6d01a2e461fa9edd250 100644 --- a/tensorflow/contrib/rnn/BUILD +++ b/tensorflow/contrib/rnn/BUILD @@ -62,21 +62,24 @@ tf_custom_op_py_library( "//tensorflow/contrib/compiler:compiler_py", "//tensorflow/contrib/layers:layers_py", "//tensorflow/contrib/util:util_py", + "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", "//tensorflow/python:clip_ops", "//tensorflow/python:embedding_ops", - "//tensorflow/python:framework", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:init_ops", - "//tensorflow/python:layers", "//tensorflow/python:math_ops", "//tensorflow/python:nn_ops", - "//tensorflow/python:partitioned_variables", "//tensorflow/python:platform", + "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:random_ops", "//tensorflow/python:rnn", "//tensorflow/python:rnn_cell", + "//tensorflow/python:session", + "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python:variables", ], ) @@ -156,6 +159,7 @@ cuda_py_tests( "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", @@ -165,6 +169,7 @@ cuda_py_tests( "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", ], shard_count = 10, ) @@ -277,6 +282,7 @@ cuda_py_tests( "//tensorflow/python:variable_scope", "//tensorflow/python:variables", ], + tags = ["no_oss"], ) tf_cc_test( @@ -379,7 +385,6 @@ py_binary( srcs_version = "PY2AND3", deps = [ "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", "//tensorflow/python:framework_ops", "//tensorflow/python:platform", "//tensorflow/python:pywrap_tensorflow", @@ -409,8 +414,5 @@ py_library( name = "benchmarking", srcs = ["python/kernel_tests/benchmarking.py"], srcs_version = "PY2AND3", - deps = [ - "//tensorflow/python:framework_ops", - "//third_party/py/numpy", - ], + deps = ["//tensorflow/python:framework_ops"], ) diff --git a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc index d82676ff7e620aef765e92137a2248c9bf1deedc..6d3758fef15e7130b740a377d8bcd41d31203299 100644 --- a/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc +++ b/tensorflow/contrib/rnn/kernels/lstm_ops_gpu.cu.cc @@ -209,7 +209,7 @@ void LSTMBlockCellFpropWithCUDA( // Use 2D blocks. The number of threads per block is equal to x * y, where x = // min(batch_size, 8) and y = 32. See above for guidance on number of // threads. - dim3 block_dim_2d(min(batch_size, 8), 32); + dim3 block_dim_2d(std::min(batch_size, 8), 32); dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast(block_dim_2d.x)), Eigen::divup(cell_size, static_cast(block_dim_2d.y))); @@ -323,7 +323,7 @@ void LSTMBlockCellBpropWithCUDA( const bool use_peephole) { const cudaStream_t& cu_stream = GetCudaStream(ctx); - dim3 block_dim_2d(min(batch_size, 8), 32); + dim3 block_dim_2d(std::min(batch_size, 8), 32); dim3 grid_dim_2d(Eigen::divup(batch_size, static_cast(block_dim_2d.x)), Eigen::divup(cell_size, static_cast(block_dim_2d.y))); diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py index 3e99b9af8521a0459f3aa313665c68d49cf0b1b9..785fc8778e29bc475deafccc60ffb260414dfa90 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_cell_test.py @@ -40,7 +40,6 @@ from tensorflow.python.ops import rnn_cell_impl from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test -from tensorflow.python.framework import test_util # pylint: enable=protected-access diff --git a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py index 2fa033632acb451762c60a68f659302102d6c3ab..9cea2ec79a982e4fb362ec564eb72b3894917842 100644 --- a/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py +++ b/tensorflow/contrib/rnn/python/kernel_tests/core_rnn_test.py @@ -25,10 +25,12 @@ from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.contrib import rnn as rnn_lib from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops as ops_lib from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl @@ -167,7 +169,7 @@ class RNNTest(test.TestCase): self.assertEqual(out.get_shape(), inp.get_shape()) self.assertEqual(out.dtype, inp.dtype) - with self.test_session(use_gpu=False) as sess: + with self.test_session(use_gpu=True) as sess: input_value = np.random.randn(batch_size, input_size) values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) @@ -202,7 +204,7 @@ class RNNTest(test.TestCase): self.assertEqual(out.get_shape().as_list(), inp.get_shape().as_list()) self.assertEqual(out.dtype, inp.dtype) - with self.test_session(use_gpu=False) as sess: + with self.test_session(use_gpu=True) as sess: input_value = np.random.randn(batch_size, input_size) values = sess.run(outputs + [state], feed_dict={inputs[0]: input_value}) full_dropout_values = sess.run(dropped_outputs, @@ -213,7 +215,7 @@ class RNNTest(test.TestCase): for d_v in full_dropout_values[:-1]: # Add 1.0 to dropped_out (all zeros) self.assertAllClose(d_v, np.ones_like(input_value)) - def _testDynamicCalculation(self, use_gpu): + def testDynamicCalculation(self): cell = Plus1RNNCell() sequence_length = array_ops.placeholder(dtypes.int64) batch_size = 2 @@ -228,7 +230,7 @@ class RNNTest(test.TestCase): cell, inputs, sequence_length=sequence_length, dtype=dtypes.float32) self.assertEqual(len(dynamic_outputs), len(inputs)) - with self.test_session(use_gpu=use_gpu) as sess: + with self.test_session(use_gpu=True) as sess: input_value = np.random.randn(batch_size, input_size) dynamic_values = sess.run( dynamic_outputs, @@ -259,10 +261,6 @@ class RNNTest(test.TestCase): np.vstack((1.0 * (1 + 1) * np.ones((input_size)), 1.0 * (2 + 1) * np.ones((input_size))))) - def testDynamicCalculation(self): - self._testDynamicCalculation(True) - self._testDynamicCalculation(False) - def _testScope(self, factory, prefix="prefix", use_outer_scope=True): with self.test_session(use_gpu=True, graph=ops_lib.Graph()): if use_outer_scope: @@ -307,12 +305,12 @@ class LSTMTest(test.TestCase): self._seed = 23489 np.random.seed(self._seed) - def _testNoProjNoSharding(self, use_gpu): + def testNoProjNoSharding(self): num_units = 3 input_size = 5 batch_size = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) cell = rnn_cell.LSTMCell( @@ -330,12 +328,12 @@ class LSTMTest(test.TestCase): input_value = np.random.randn(batch_size, input_size) sess.run(outputs, feed_dict={inputs[0]: input_value}) - def _testCellClipping(self, use_gpu): + def testCellClipping(self): num_units = 3 input_size = 5 batch_size = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) cell = rnn_cell.LSTMCell( @@ -361,12 +359,12 @@ class LSTMTest(test.TestCase): # if cell c is clipped to 0, tanh(c) = 0 => m==0 self.assertAllEqual(value, np.zeros((batch_size, num_units))) - def _testNoProjNoShardingSimpleStateSaver(self, use_gpu): + def testNoProjNoShardingSimpleStateSaver(self): num_units = 3 input_size = 5 batch_size = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) state_saver = TestStateSaver(batch_size, 2 * num_units) @@ -491,13 +489,13 @@ class LSTMTest(test.TestCase): self.assertAllEqual(last_states[i], named_saved_states[flat_state_names[i]]) - def _testProjNoSharding(self, use_gpu): + def testProjNoSharding(self): num_units = 3 input_size = 5 batch_size = 2 num_proj = 4 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) inputs = max_length * [ @@ -582,7 +580,7 @@ class LSTMTest(test.TestCase): state_tuple_v = sess.run(state_tuple, feed_dict={inputs[0]: input_value}) self.assertAllEqual(state_notuple_v, np.hstack(state_tuple_v)) - def _testProjSharding(self, use_gpu): + def testProjSharding(self): num_units = 3 input_size = 5 batch_size = 2 @@ -590,7 +588,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) @@ -616,7 +614,7 @@ class LSTMTest(test.TestCase): input_value = np.random.randn(batch_size, input_size) sess.run(outputs, feed_dict={inputs[0]: input_value}) - def _testDoubleInput(self, use_gpu): + def testDoubleInput(self): num_units = 3 input_size = 5 batch_size = 2 @@ -624,7 +622,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer(-1, 1, seed=self._seed) inputs = max_length * [ array_ops.placeholder( @@ -653,7 +651,7 @@ class LSTMTest(test.TestCase): values = sess.run(outputs, feed_dict={inputs[0]: input_value}) self.assertEqual(values[0].dtype, input_value.dtype) - def _testShardNoShardEquivalentOutput(self, use_gpu): + def testShardNoShardEquivalentOutput(self): num_units = 3 input_size = 5 batch_size = 2 @@ -661,7 +659,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: inputs = max_length * [ array_ops.placeholder( dtypes.float32, shape=(None, input_size)) @@ -708,7 +706,7 @@ class LSTMTest(test.TestCase): for (s_noshard, s_shard) in zip(state_values_noshard, state_values_shard): self.assertAllClose(s_noshard, s_shard, atol=1e-3) - def _testDoubleInputWithDropoutAndDynamicCalculation(self, use_gpu): + def testDoubleInputWithDropoutAndDynamicCalculation(self): """Smoke test for using LSTM with doubles, dropout, dynamic calculation.""" num_units = 3 @@ -718,7 +716,7 @@ class LSTMTest(test.TestCase): num_proj_shards = 3 num_unit_shards = 2 max_length = 8 - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: sequence_length = array_ops.placeholder(dtypes.int64) initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) @@ -843,44 +841,13 @@ class LSTMTest(test.TestCase): for out0, out1 in zip(outputs0_values, outputs1_values): self.assertAllEqual(out0, out1) - def testNoProjNoShardingSimpleStateSaver(self): - self._testNoProjNoShardingSimpleStateSaver(use_gpu=False) - self._testNoProjNoShardingSimpleStateSaver(use_gpu=True) - - def testNoProjNoSharding(self): - self._testNoProjNoSharding(use_gpu=False) - self._testNoProjNoSharding(use_gpu=True) - - def testCellClipping(self): - self._testCellClipping(use_gpu=False) - self._testCellClipping(use_gpu=True) - - def testProjNoSharding(self): - self._testProjNoSharding(use_gpu=False) - self._testProjNoSharding(use_gpu=True) - - def testProjSharding(self): - self._testProjSharding(use_gpu=False) - self._testProjSharding(use_gpu=True) - - def testShardNoShardEquivalentOutput(self): - self._testShardNoShardEquivalentOutput(use_gpu=False) - self._testShardNoShardEquivalentOutput(use_gpu=True) - - def testDoubleInput(self): - self._testDoubleInput(use_gpu=False) - self._testDoubleInput(use_gpu=True) - - def testDoubleInputWithDropoutAndDynamicCalculation(self): - self._testDoubleInputWithDropoutAndDynamicCalculation(use_gpu=False) - self._testDoubleInputWithDropoutAndDynamicCalculation(use_gpu=True) - def testDynamicRNNAllowsUnknownTimeDimension(self): inputs = array_ops.placeholder(dtypes.float32, shape=[1, None, 20]) cell = rnn_cell.GRUCell(30) # Smoke test, this should not raise an error rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) + @test_util.run_in_graph_and_eager_modes() def testDynamicRNNWithTupleStates(self): num_units = 3 input_size = 5 @@ -888,13 +855,20 @@ class LSTMTest(test.TestCase): num_proj = 4 max_length = 8 sequence_length = [4, 6] + in_graph_mode = context.in_graph_mode() with self.test_session(graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, shape=(None, input_size)) - ] + if in_graph_mode: + inputs = max_length * [ + array_ops.placeholder( + dtypes.float32, shape=(None, input_size)) + ] + else: + inputs = max_length * [ + constant_op.constant( + np.random.randn(batch_size, input_size).astype(np.float32)) + ] inputs_c = array_ops.stack(inputs) cell = rnn_cell.LSTMCell( num_units, @@ -924,21 +898,34 @@ class LSTMTest(test.TestCase): self.assertEqual(state_dynamic[0], state_dynamic.c) self.assertEqual(state_dynamic[1], state_dynamic.h) - variables_lib.global_variables_initializer().run() - - input_value = np.random.randn(batch_size, input_size) - outputs_static_v = sess.run(outputs_static, - feed_dict={inputs[0]: input_value}) - outputs_dynamic_v = sess.run(outputs_dynamic, - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(outputs_static_v, outputs_dynamic_v) - - state_static_v = sess.run(state_static, - feed_dict={inputs[0]: input_value}) - state_dynamic_v = sess.run(state_dynamic, - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(np.hstack(state_static_v), np.hstack(state_dynamic_v)) + if in_graph_mode: + variables_lib.global_variables_initializer().run() + input_value = np.random.randn(batch_size, input_size) + outputs_static = sess.run( + outputs_static, feed_dict={ + inputs[0]: input_value + }) + outputs_dynamic = sess.run( + outputs_dynamic, feed_dict={ + inputs[0]: input_value + }) + state_static = sess.run( + state_static, feed_dict={ + inputs[0]: input_value + }) + state_dynamic = sess.run( + state_dynamic, feed_dict={ + inputs[0]: input_value + }) + + if in_graph_mode: + self.assertAllEqual(outputs_static, outputs_dynamic) + else: + self.assertAllEqual( + array_ops.stack(outputs_static).numpy(), outputs_dynamic.numpy()) + self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic)) + @test_util.run_in_graph_and_eager_modes() def testDynamicRNNWithNestedTupleStates(self): num_units = 3 input_size = 5 @@ -946,13 +933,20 @@ class LSTMTest(test.TestCase): num_proj = 4 max_length = 8 sequence_length = [4, 6] + in_graph_mode = context.in_graph_mode() with self.test_session(graph=ops_lib.Graph()) as sess: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - inputs = max_length * [ - array_ops.placeholder( - dtypes.float32, shape=(None, input_size)) - ] + if in_graph_mode: + inputs = max_length * [ + array_ops.placeholder( + dtypes.float32, shape=(None, input_size)) + ] + else: + inputs = max_length * [ + constant_op.constant( + np.random.randn(batch_size, input_size).astype(np.float32)) + ] inputs_c = array_ops.stack(inputs) def _cell(i): @@ -993,43 +987,58 @@ class LSTMTest(test.TestCase): sequence_length=sequence_length, scope=scope) - variables_lib.global_variables_initializer().run() - - input_value = np.random.randn(batch_size, input_size) - outputs_static_v = sess.run(outputs_static, - feed_dict={inputs[0]: input_value}) - outputs_dynamic_v = sess.run(outputs_dynamic, - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(outputs_static_v, outputs_dynamic_v) - - state_static_v = sess.run(nest.flatten(state_static), - feed_dict={inputs[0]: input_value}) - state_dynamic_v = sess.run(nest.flatten(state_dynamic), - feed_dict={inputs[0]: input_value}) - self.assertAllEqual(np.hstack(state_static_v), np.hstack(state_dynamic_v)) + if in_graph_mode: + input_value = np.random.randn(batch_size, input_size) + variables_lib.global_variables_initializer().run() + outputs_static = sess.run( + outputs_static, feed_dict={ + inputs[0]: input_value + }) + outputs_dynamic = sess.run( + outputs_dynamic, feed_dict={ + inputs[0]: input_value + }) + state_static = sess.run( + nest.flatten(state_static), feed_dict={ + inputs[0]: input_value + }) + state_dynamic = sess.run( + nest.flatten(state_dynamic), feed_dict={ + inputs[0]: input_value + }) + + if in_graph_mode: + self.assertAllEqual(outputs_static, outputs_dynamic) + else: + self.assertAllEqual( + array_ops.stack(outputs_static).numpy(), outputs_dynamic.numpy()) + state_static = [s.numpy() for s in nest.flatten(state_static)] + state_dynamic = [s.numpy() for s in nest.flatten(state_dynamic)] + self.assertAllEqual(np.hstack(state_static), np.hstack(state_dynamic)) - def _testDynamicEquivalentToStaticRNN(self, use_gpu, use_sequence_length): + def _testDynamicEquivalentToStaticRNN(self, use_sequence_length): time_steps = 8 num_units = 3 num_proj = 4 input_size = 5 batch_size = 2 - input_values = np.random.randn(time_steps, batch_size, input_size) + input_values = np.random.randn(time_steps, batch_size, input_size).astype( + np.float32) if use_sequence_length: sequence_length = np.random.randint(0, time_steps, size=batch_size) else: sequence_length = None - ########### Step 1: Run static graph and generate readouts - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: - concat_inputs = array_ops.placeholder( - dtypes.float32, shape=(time_steps, batch_size, input_size)) - inputs = array_ops.unstack(concat_inputs) + in_graph_mode = context.in_graph_mode() + + # TODO(b/68017812): Eager ignores operation seeds, so we need to create a + # single cell and reuse it across the static and dynamic RNNs. Remove this + # special case once is fixed. + if not in_graph_mode: initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - cell = rnn_cell.LSTMCell( num_units, use_peepholes=True, @@ -1037,63 +1046,85 @@ class LSTMTest(test.TestCase): num_proj=num_proj, state_is_tuple=False) + ########### Step 1: Run static graph and generate readouts + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: + if in_graph_mode: + concat_inputs = array_ops.placeholder( + dtypes.float32, shape=(time_steps, batch_size, input_size)) + else: + concat_inputs = constant_op.constant(input_values) + inputs = array_ops.unstack(concat_inputs) + initializer = init_ops.random_uniform_initializer( + -0.01, 0.01, seed=self._seed) + + # TODO(akshayka): Remove special case once b/68017812 is fixed. + if in_graph_mode: + cell = rnn_cell.LSTMCell( + num_units, + use_peepholes=True, + initializer=initializer, + num_proj=num_proj, + state_is_tuple=False) + with variable_scope.variable_scope("dynamic_scope"): outputs_static, state_static = rnn.static_rnn( cell, inputs, sequence_length=sequence_length, dtype=dtypes.float32) - feeds = {concat_inputs: input_values} - - # Initialize - variables_lib.global_variables_initializer().run(feed_dict=feeds) - - # Generate gradients of sum of outputs w.r.t. inputs - static_gradients = gradients_impl.gradients( - outputs_static + [state_static], [concat_inputs]) - - # Generate gradients of individual outputs w.r.t. inputs - static_individual_gradients = nest.flatten([ - gradients_impl.gradients(y, [concat_inputs]) - for y in [outputs_static[0], outputs_static[-1], state_static] - ]) - - # Generate gradients of individual variables w.r.t. inputs - trainable_variables = ops_lib.get_collection( - ops_lib.GraphKeys.TRAINABLE_VARIABLES) - assert len(trainable_variables) > 1, ("Count of trainable variables: %d" % - len(trainable_variables)) - # pylint: disable=bad-builtin - static_individual_variable_gradients = nest.flatten([ - gradients_impl.gradients(y, trainable_variables) - for y in [outputs_static[0], outputs_static[-1], state_static] - ]) - - # Test forward pass - values_static = sess.run(outputs_static, feed_dict=feeds) - (state_value_static,) = sess.run((state_static,), feed_dict=feeds) - - # Test gradients to inputs and variables w.r.t. outputs & final state - static_grad_values = sess.run(static_gradients, feed_dict=feeds) - - static_individual_grad_values = sess.run(static_individual_gradients, - feed_dict=feeds) - - static_individual_var_grad_values = sess.run( - static_individual_variable_gradients, feed_dict=feeds) + if in_graph_mode: + # Generate gradients and run sessions to obtain outputs + feeds = {concat_inputs: input_values} + # Initialize + variables_lib.global_variables_initializer().run(feed_dict=feeds) + # Generate gradients of sum of outputs w.r.t. inputs + static_gradients = gradients_impl.gradients( + outputs_static + [state_static], [concat_inputs]) + # Generate gradients of individual outputs w.r.t. inputs + static_individual_gradients = nest.flatten([ + gradients_impl.gradients(y, [concat_inputs]) + for y in [outputs_static[0], outputs_static[-1], state_static] + ]) + # Generate gradients of individual variables w.r.t. inputs + trainable_variables = ops_lib.get_collection( + ops_lib.GraphKeys.TRAINABLE_VARIABLES) + assert len(trainable_variables) > 1, ( + "Count of trainable variables: %d" % len(trainable_variables)) + # pylint: disable=bad-builtin + static_individual_variable_gradients = nest.flatten([ + gradients_impl.gradients(y, trainable_variables) + for y in [outputs_static[0], outputs_static[-1], state_static] + ]) + # Test forward pass + values_static = sess.run(outputs_static, feed_dict=feeds) + (state_value_static,) = sess.run((state_static,), feed_dict=feeds) + + # Test gradients to inputs and variables w.r.t. outputs & final state + static_grad_values = sess.run(static_gradients, feed_dict=feeds) + + static_individual_grad_values = sess.run(static_individual_gradients, + feed_dict=feeds) + + static_individual_var_grad_values = sess.run( + static_individual_variable_gradients, feed_dict=feeds) ########## Step 2: Run dynamic graph and generate readouts - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: - concat_inputs = array_ops.placeholder( - dtypes.float32, shape=(time_steps, batch_size, input_size)) - inputs = array_ops.unstack(concat_inputs) + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: + if in_graph_mode: + concat_inputs = array_ops.placeholder( + dtypes.float32, shape=(time_steps, batch_size, input_size)) + else: + concat_inputs = constant_op.constant(input_values) initializer = init_ops.random_uniform_initializer( -0.01, 0.01, seed=self._seed) - cell = rnn_cell.LSTMCell( - num_units, - use_peepholes=True, - initializer=initializer, - num_proj=num_proj, - state_is_tuple=False) + # TODO(akshayka): Remove this special case once b/68017812 is + # fixed. + if in_graph_mode: + cell = rnn_cell.LSTMCell( + num_units, + use_peepholes=True, + initializer=initializer, + num_proj=num_proj, + state_is_tuple=False) with variable_scope.variable_scope("dynamic_scope"): outputs_dynamic, state_dynamic = rnn.dynamic_rnn( @@ -1104,81 +1135,86 @@ class LSTMTest(test.TestCase): dtype=dtypes.float32) split_outputs_dynamic = array_ops.unstack(outputs_dynamic, time_steps) - feeds = {concat_inputs: input_values} + if in_graph_mode: + feeds = {concat_inputs: input_values} - # Initialize - variables_lib.global_variables_initializer().run(feed_dict=feeds) + # Initialize + variables_lib.global_variables_initializer().run(feed_dict=feeds) + + # Generate gradients of sum of outputs w.r.t. inputs + dynamic_gradients = gradients_impl.gradients( + split_outputs_dynamic + [state_dynamic], [concat_inputs]) + + # Generate gradients of several individual outputs w.r.t. inputs + dynamic_individual_gradients = nest.flatten([ + gradients_impl.gradients(y, [concat_inputs]) + for y in + [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] + ]) + + # Generate gradients of individual variables w.r.t. inputs + trainable_variables = ops_lib.get_collection( + ops_lib.GraphKeys.TRAINABLE_VARIABLES) + assert len(trainable_variables) > 1, ( + "Count of trainable variables: %d" % len(trainable_variables)) + dynamic_individual_variable_gradients = nest.flatten([ + gradients_impl.gradients(y, trainable_variables) + for y in + [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] + ]) + + # Test forward pass + values_dynamic = sess.run(split_outputs_dynamic, feed_dict=feeds) + (state_value_dynamic,) = sess.run((state_dynamic,), feed_dict=feeds) - # Generate gradients of sum of outputs w.r.t. inputs - dynamic_gradients = gradients_impl.gradients( - split_outputs_dynamic + [state_dynamic], [concat_inputs]) - - # Generate gradients of several individual outputs w.r.t. inputs - dynamic_individual_gradients = nest.flatten([ - gradients_impl.gradients(y, [concat_inputs]) - for y in - [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] - ]) - - # Generate gradients of individual variables w.r.t. inputs - trainable_variables = ops_lib.get_collection( - ops_lib.GraphKeys.TRAINABLE_VARIABLES) - assert len(trainable_variables) > 1, ("Count of trainable variables: %d" % - len(trainable_variables)) - dynamic_individual_variable_gradients = nest.flatten([ - gradients_impl.gradients(y, trainable_variables) - for y in - [split_outputs_dynamic[0], split_outputs_dynamic[-1], state_dynamic] - ]) - - # Test forward pass - values_dynamic = sess.run(split_outputs_dynamic, feed_dict=feeds) - (state_value_dynamic,) = sess.run((state_dynamic,), feed_dict=feeds) - - # Test gradients to inputs and variables w.r.t. outputs & final state - dynamic_grad_values = sess.run(dynamic_gradients, feed_dict=feeds) - - dynamic_individual_grad_values = sess.run(dynamic_individual_gradients, - feed_dict=feeds) - - dynamic_individual_var_grad_values = sess.run( - dynamic_individual_variable_gradients, feed_dict=feeds) + # Test gradients to inputs and variables w.r.t. outputs & final state + dynamic_grad_values = sess.run(dynamic_gradients, feed_dict=feeds) + + dynamic_individual_grad_values = sess.run(dynamic_individual_gradients, + feed_dict=feeds) + + dynamic_individual_var_grad_values = sess.run( + dynamic_individual_variable_gradients, feed_dict=feeds) ######### Step 3: Comparisons + if not in_graph_mode: + values_static = outputs_static + values_dynamic = split_outputs_dynamic + state_value_static = state_static + state_value_dynamic = state_dynamic + self.assertEqual(len(values_static), len(values_dynamic)) for (value_static, value_dynamic) in zip(values_static, values_dynamic): self.assertAllEqual(value_static, value_dynamic) self.assertAllEqual(state_value_static, state_value_dynamic) - self.assertAllEqual(static_grad_values, dynamic_grad_values) + if in_graph_mode: + + self.assertAllEqual(static_grad_values, dynamic_grad_values) - self.assertEqual( - len(static_individual_grad_values), len(dynamic_individual_grad_values)) - self.assertEqual( - len(static_individual_var_grad_values), - len(dynamic_individual_var_grad_values)) + self.assertEqual( + len(static_individual_grad_values), + len(dynamic_individual_grad_values)) + self.assertEqual( + len(static_individual_var_grad_values), + len(dynamic_individual_var_grad_values)) - for i, (a, b) in enumerate( - zip(static_individual_grad_values, dynamic_individual_grad_values)): - tf_logging.info("Comparing individual gradients iteration %d" % i) - self.assertAllEqual(a, b) + for i, (a, b) in enumerate( + zip(static_individual_grad_values, dynamic_individual_grad_values)): + tf_logging.info("Comparing individual gradients iteration %d" % i) + self.assertAllEqual(a, b) - for i, (a, b) in enumerate( - zip(static_individual_var_grad_values, - dynamic_individual_var_grad_values)): - tf_logging.info("Comparing individual variable gradients iteration %d" % - i) - self.assertAllEqual(a, b) + for i, (a, b) in enumerate( + zip(static_individual_var_grad_values, + dynamic_individual_var_grad_values)): + tf_logging.info("Comparing individual variable gradients iteration %d" % + i) + self.assertAllEqual(a, b) + @test_util.run_in_graph_and_eager_modes() def testDynamicEquivalentToStaticRNN(self): - self._testDynamicEquivalentToStaticRNN( - use_gpu=False, use_sequence_length=False) - self._testDynamicEquivalentToStaticRNN( - use_gpu=True, use_sequence_length=False) - self._testDynamicEquivalentToStaticRNN( - use_gpu=False, use_sequence_length=True) - self._testDynamicEquivalentToStaticRNN( - use_gpu=True, use_sequence_length=True) + self._testDynamicEquivalentToStaticRNN(use_sequence_length=False) + self._testDynamicEquivalentToStaticRNN(use_sequence_length=False) class BidirectionalRNNTest(test.TestCase): @@ -1188,7 +1224,6 @@ class BidirectionalRNNTest(test.TestCase): np.random.seed(self._seed) def _createBidirectionalRNN(self, - use_gpu, use_shape, use_sequence_length, scope=None): @@ -1227,10 +1262,10 @@ class BidirectionalRNNTest(test.TestCase): return input_value, inputs, outputs, state_fw, state_bw, sequence_length - def _testBidirectionalRNN(self, use_gpu, use_shape): - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + def _testBidirectionalRNN(self, use_shape): + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createBidirectionalRNN(use_gpu, use_shape, True)) + self._createBidirectionalRNN(use_shape, True)) variables_lib.global_variables_initializer().run() # Run with pre-specified sequence length of 2, 3 out, s_fw, s_bw = sess.run( @@ -1272,10 +1307,10 @@ class BidirectionalRNNTest(test.TestCase): # exactly the same self.assertAllClose(s_fw, s_bw) - def _testBidirectionalRNNWithoutSequenceLength(self, use_gpu, use_shape): - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + def _testBidirectionalRNNWithoutSequenceLength(self, use_shape): + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: input_value, inputs, outputs, state_fw, state_bw, _ = ( - self._createBidirectionalRNN(use_gpu, use_shape, False)) + self._createBidirectionalRNN(use_shape, False)) variables_lib.global_variables_initializer().run() out, s_fw, s_bw = sess.run([outputs, state_fw, state_bw], feed_dict={inputs[0]: input_value}) @@ -1302,23 +1337,14 @@ class BidirectionalRNNTest(test.TestCase): self.assertAllClose(s_fw, s_bw) def testBidirectionalRNN(self): - self._testBidirectionalRNN(use_gpu=False, use_shape=False) - self._testBidirectionalRNN(use_gpu=True, use_shape=False) - self._testBidirectionalRNN(use_gpu=False, use_shape=True) - self._testBidirectionalRNN(use_gpu=True, use_shape=True) + self._testBidirectionalRNN(use_shape=False) + self._testBidirectionalRNN(use_shape=True) def testBidirectionalRNNWithoutSequenceLength(self): - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=False, use_shape=False) - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=True, use_shape=False) - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=False, use_shape=True) - self._testBidirectionalRNNWithoutSequenceLength( - use_gpu=True, use_shape=True) + self._testBidirectionalRNNWithoutSequenceLength(use_shape=False) + self._testBidirectionalRNNWithoutSequenceLength(use_shape=True) def _createBidirectionalDynamicRNN(self, - use_gpu, use_shape, use_state_tuple, use_time_major, @@ -1366,11 +1392,11 @@ class BidirectionalRNNTest(test.TestCase): return input_value, inputs, outputs, state_fw, state_bw, sequence_length - def _testBidirectionalDynamicRNN(self, use_gpu, use_shape, use_state_tuple, + def _testBidirectionalDynamicRNN(self, use_shape, use_state_tuple, use_time_major, use_sequence_length): - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: input_value, inputs, outputs, state_fw, state_bw, sequence_length = ( - self._createBidirectionalDynamicRNN(use_gpu, use_shape, + self._createBidirectionalDynamicRNN(use_shape, use_state_tuple, use_time_major, use_sequence_length)) variables_lib.global_variables_initializer().run() @@ -1435,14 +1461,13 @@ class BidirectionalRNNTest(test.TestCase): def testBidirectionalDynamicRNN(self): # Generate 2^5 option values # from [True, True, True, True, True] to [False, False, False, False, False] - options = itertools.product([True, False], repeat=5) + options = itertools.product([True, False], repeat=4) for option in options: self._testBidirectionalDynamicRNN( - use_gpu=option[0], - use_shape=option[1], - use_state_tuple=option[2], - use_time_major=option[3], - use_sequence_length=option[4]) + use_shape=option[0], + use_state_tuple=option[1], + use_time_major=option[2], + use_sequence_length=option[3]) def _testScope(self, factory, prefix="prefix", use_outer_scope=True): # REMARKS: factory(scope) is a function accepting a scope @@ -1471,7 +1496,7 @@ class BidirectionalRNNTest(test.TestCase): def factory(scope): return self._createBidirectionalRNN( - use_gpu=True, use_shape=True, use_sequence_length=True, scope=scope) + use_shape=True, use_sequence_length=True, scope=scope) self._testScope(factory, use_outer_scope=True) self._testScope(factory, use_outer_scope=False) @@ -1483,7 +1508,6 @@ class BidirectionalRNNTest(test.TestCase): def factory(scope): return self._createBidirectionalDynamicRNN( - use_gpu=True, use_shape=True, use_state_tuple=True, use_sequence_length=True, @@ -1761,7 +1785,7 @@ class GRUTest(test.TestCase): self._seed = 23489 np.random.seed(self._seed) - def _testDynamic(self, use_gpu): + def testDynamic(self): time_steps = 8 num_units = 3 input_size = 5 @@ -1771,7 +1795,7 @@ class GRUTest(test.TestCase): sequence_length = np.random.randint(0, time_steps, size=batch_size) - with self.test_session(use_gpu=use_gpu, graph=ops_lib.Graph()) as sess: + with self.test_session(use_gpu=True, graph=ops_lib.Graph()) as sess: concat_inputs = array_ops.placeholder( dtypes.float32, shape=(time_steps, batch_size, input_size)) @@ -1792,10 +1816,6 @@ class GRUTest(test.TestCase): sess.run([outputs_dynamic, state_dynamic], feed_dict=feeds) - def testDynamic(self): - self._testDynamic(use_gpu=False) - self._testDynamic(use_gpu=True) - def _testScope(self, factory, prefix="prefix", use_outer_scope=True): with self.test_session(use_gpu=True, graph=ops_lib.Graph()): if use_outer_scope: diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD index a82ee6ac41ed3f81bd96c61dafb2144c41b07065..20be819e07d0e47a0b24b5cc2548727322093e50 100644 --- a/tensorflow/contrib/saved_model/BUILD +++ b/tensorflow/contrib/saved_model/BUILD @@ -37,9 +37,14 @@ py_library( visibility = ["//visibility:public"], deps = [ "//tensorflow/core:protos_all_py", + "//tensorflow/python:framework_ops", "//tensorflow/python:lib", "//tensorflow/python:util", + "//tensorflow/python/saved_model:builder", "//tensorflow/python/saved_model:constants", + "//tensorflow/python/saved_model:signature_constants", + "//tensorflow/python/saved_model:signature_def_utils", + "//tensorflow/python/saved_model:tag_constants", ], ) @@ -85,10 +90,11 @@ py_test( deps = [ ":saved_model_py", "//tensorflow/python:client_testlib", - "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", - "//tensorflow/python/saved_model", + "//tensorflow/python:variables", + "//tensorflow/python/saved_model:loader", "//tensorflow/python/saved_model:signature_constants", + "//tensorflow/python/saved_model:tag_constants", ], ) diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD index f1e39a137322711efacda02abd3c13f528981bc1..ab80c68b1a8e4ff151494e393b68c460846fa8fe 100644 --- a/tensorflow/contrib/seq2seq/BUILD +++ b/tensorflow/contrib/seq2seq/BUILD @@ -33,18 +33,31 @@ tf_custom_op_py_library( "//tensorflow/contrib/distributions:distributions_py", "//tensorflow/contrib/layers:layers_py", "//tensorflow/contrib/rnn:rnn_py", + "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", + "//tensorflow/python:check_ops", + "//tensorflow/python:clip_ops", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:embedding_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:functional_ops", + "//tensorflow/python:init_ops", + "//tensorflow/python:layers", + "//tensorflow/python:layers_base", "//tensorflow/python:math_ops", + "//tensorflow/python:nn_ops", "//tensorflow/python:platform", + "//tensorflow/python:random_ops", "//tensorflow/python:rnn", "//tensorflow/python:rnn_cell", "//tensorflow/python:script_ops", "//tensorflow/python:tensor_array_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//tensorflow/python:variable_scope", + "//tensorflow/python/ops/distributions", "//third_party/py/numpy", + "@six_archive//:six", ], ) diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc index aab0f3f4947388741765b268094b4136d356a457..64973ccccdc962757a727d7183bd70e94edcfd1b 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc @@ -49,40 +49,46 @@ class GatherTreeOp : public OpKernel { const Device& device = ctx->eigen_device(); const Tensor& step_ids = ctx->input(0); const Tensor& parent_ids = ctx->input(1); - const Tensor& sequence_length = ctx->input(2); + const Tensor& max_sequence_lengths = ctx->input(2); + const Tensor& end_token = ctx->input(3); const TensorShape& step_ids_shape = step_ids.shape(); OP_REQUIRES( ctx, step_ids_shape.dims() == 3, errors::InvalidArgument("step_ids must be a 3-tensor, saw shape: ", step_ids_shape.DebugString())); - OP_REQUIRES( - ctx, TensorShapeUtils::IsMatrix(sequence_length.shape()), - errors::InvalidArgument("sequence_length must be a matrix, saw shape: ", - sequence_length.shape().DebugString())); - OP_REQUIRES(ctx, sequence_length.dim_size(0) == step_ids_shape.dim_size(1), - errors::InvalidArgument( - "Inconsistent batch sizes: sequence_length.shape[0] (", - sequence_length.dim_size(0), ") != ", "step_ids.shape[1] (", - step_ids_shape.dim_size(1), ")")); - OP_REQUIRES(ctx, sequence_length.dim_size(1) == step_ids_shape.dim_size(2), + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(max_sequence_lengths.shape()), errors::InvalidArgument( - "Inconsistent batch sizes: sequence_length.shape[1] (", - sequence_length.dim_size(1), ") != ", "step_ids.shape[2] (", - step_ids_shape.dim_size(2), ")")); + "max_sequence_lengths must be a vector, saw shape: ", + max_sequence_lengths.shape().DebugString())); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(end_token.shape()), + errors::InvalidArgument("end_token must be a scalar, saw shape: ", + end_token.shape().DebugString())); OP_REQUIRES( ctx, step_ids_shape == parent_ids.shape(), errors::InvalidArgument( "step_ids.shape must match parent_ids.shape. but shapes are: ", step_ids_shape.DebugString(), " and ", parent_ids.shape().DebugString())); + OP_REQUIRES( + ctx, + step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0), + errors::InvalidArgument("batch size dimensions step_ids.shape[1] and " + "max_seqeuence_lengths.shape[0] must match. " + "but shapes are: ", + step_ids_shape.DebugString(), " and ", + max_sequence_lengths.shape().DebugString())); Tensor* beams; OP_REQUIRES_OK(ctx, ctx->allocate_output(0, step_ids_shape, &beams)); typename TTypes::ConstTensor step_ids_t = step_ids.tensor(); typename TTypes::ConstTensor parent_ids_t = parent_ids.tensor(); - typename TTypes::ConstMatrix seq_len_t = sequence_length.matrix(); + typename TTypes::ConstVec max_seq_lens_t = + max_sequence_lengths.vec(); + typename TTypes::ConstScalar end_token_t = end_token.scalar(); typename TTypes::Tensor beams_t = beams->tensor(); + const T end_token_value = end_token_t(); functor::GatherTree()(ctx, device, step_ids_t, parent_ids_t, - seq_len_t, beams_t); + max_seq_lens_t, end_token_value, beams_t); } }; @@ -99,27 +105,29 @@ namespace functor { template <> struct GatherTree { void operator()(OpKernelContext* ctx, const CPUDevice& d, - typename TTypes::ConstTensor step_ids, - typename TTypes::ConstTensor parent_ids, - typename TTypes::ConstMatrix sequence_length, - typename TTypes::Tensor beams) { - const int64 max_time = parent_ids.dimension(0); - const int64 batch_size = parent_ids.dimension(1); - const int64 beam_width = parent_ids.dimension(2); - beams.setConstant(-1); - - auto DoWork = [&, ctx](int start_batch_beam, int limit_batch_beam) { + TTypes::ConstTensor step_ids, + TTypes::ConstTensor parent_ids, + TTypes::ConstVec max_sequence_lengths, + const int32 end_token, TTypes::Tensor beams) { + const int32 max_time = parent_ids.dimension(0); + const int32 batch_size = parent_ids.dimension(1); + const int32 beam_width = parent_ids.dimension(2); + beams.setConstant(end_token); + + auto DoWork = [&, ctx, end_token](int start_batch_beam, + int limit_batch_beam) { for (int32 i = start_batch_beam; i < limit_batch_beam; ++i) { const int32 batch = i / beam_width; const int32 beam = i % beam_width; - int32 seq_len_b = sequence_length(batch, beam); - if (seq_len_b <= 0) { + const int32 max_seq_len_b = + Eigen::numext::mini(max_time, max_sequence_lengths(batch)); + if (max_seq_len_b <= 0) { continue; } - beams(seq_len_b - 1, batch, beam) = - step_ids(seq_len_b - 1, batch, beam); - int32 parent = parent_ids(seq_len_b - 1, batch, beam); - for (int32 level = seq_len_b - 2; level >= 0; --level) { + beams(max_seq_len_b - 1, batch, beam) = + step_ids(max_seq_len_b - 1, batch, beam); + int32 parent = parent_ids(max_seq_len_b - 1, batch, beam); + for (int32 level = max_seq_len_b - 2; level >= 0; --level) { if (parent < 0 || parent > beam_width) { ctx->SetStatus( errors::InvalidArgument("Saw invalid parent id ", parent, @@ -130,6 +138,17 @@ struct GatherTree { beams(level, batch, beam) = step_ids(level, batch, parent); parent = parent_ids(level, batch, parent); } + // Not necessary when using a BeamSearchDecoder, but necessary + // when a user feeds in possibly broken trajectory (i.e., non-eos + // entries in a beam following eos entries). + bool finished = false; + for (int32 time = 0; time < max_seq_len_b; ++time) { + if (finished) { + beams(time, batch, beam) = end_token; + } else if (beams(time, batch, beam) == end_token) { + finished = true; + } + } } }; // Guesstimate of cost; ~5 lookup/store/compare per inner beam @@ -137,7 +156,7 @@ struct GatherTree { const int64 batch_beam_cost = Eigen::TensorOpCost::DivCost() + 6 * Eigen::TensorOpCost::AddCost() + - max_time * (5 * Eigen::TensorOpCost::AddCost()); + 2 * max_time * (5 * Eigen::TensorOpCost::AddCost()); auto worker_threads = *(ctx->device()->tensorflow_cpu_worker_threads()); Shard(worker_threads.num_threads, worker_threads.workers, batch_size * beam_width, batch_beam_cost, DoWork); @@ -148,24 +167,26 @@ struct GatherTree { #if GOOGLE_CUDA namespace functor { -#define DECLARE_GPU_SPEC(T) \ - template <> \ - void GatherTree::operator()( \ - OpKernelContext* ctx, const GPUDevice& d, \ - typename TTypes::ConstTensor step_ids, \ - typename TTypes::ConstTensor parent_ids, \ - typename TTypes::ConstMatrix sequence_length, \ - typename TTypes::Tensor beams); \ +#define DECLARE_GPU_SPEC(T) \ + template <> \ + void GatherTree::operator()( \ + OpKernelContext* ctx, const GPUDevice& d, \ + typename TTypes::ConstTensor step_ids, \ + typename TTypes::ConstTensor parent_ids, \ + TTypes::ConstVec max_sequence_lengths, const T end_token, \ + typename TTypes::Tensor beams); \ extern template struct GatherTree; DECLARE_GPU_SPEC(int32); #undef DECLARE_GPU_SPEC } // end namespace functor -#define REGISTER_GPU_KERNEL(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("GatherTree").Device(DEVICE_GPU).TypeConstraint("T"), \ - GatherTreeOp); +#define REGISTER_GPU_KERNEL(T) \ + REGISTER_KERNEL_BUILDER(Name("GatherTree") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("end_token"), \ + GatherTreeOp); REGISTER_GPU_KERNEL(int32); #undef REGISTER_GPU_KERNEL diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h index 124d07264e75ac4ce7739dd3291abdabbb40a58f..693b02dc437afdf14c38e4224c5469bb3e569540 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.h @@ -31,8 +31,8 @@ struct GatherTree { void operator()(OpKernelContext* ctx, const Device& d, typename TTypes::ConstTensor step_ids, typename TTypes::ConstTensor parent_ids, - typename TTypes::ConstMatrix sequence_length, - typename TTypes::Tensor beams); + TTypes::ConstVec max_sequence_lengths, + const T end_token, typename TTypes::Tensor beams); }; } // namespace functor diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc index ee68b55d20214c207597750e083a63e94ebdc0a0..bc28d492fe1a25afe0d0783539aa9e759e7b703f 100644 --- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc +++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops_gpu.cu.cc @@ -29,30 +29,50 @@ template __global__ void GatherTreeOpKernel(const int32 batch_size, const int32 max_time, const int32 beam_width, const T* step_ids, const T* parent_ids, - const T* sequence_length, T* beams) { + const int32* max_sequence_lengths, + const T end_token, T* beams) { CUDA_1D_KERNEL_LOOP(i, batch_size * beam_width) { const int32 batch = i / beam_width; const int32 beam = i % beam_width; - const int32 seq_len_b = ldg(sequence_length + batch * beam_width + beam); - if (seq_len_b <= 0) continue; + const int32 max_seq_len_b = + Eigen::numext::mini(max_time, ldg(max_sequence_lengths + batch)); + if (max_seq_len_b <= 0) { + continue; + } #define GET_IX(time_ix, beam_ix) \ (batch_size * beam_width * (time_ix) + beam_width * batch + (beam_ix)) - const int32 initial_beam_ix = GET_IX(seq_len_b - 1, beam); + const int32 initial_beam_ix = GET_IX(max_seq_len_b - 1, beam); beams[initial_beam_ix] = ldg(step_ids + initial_beam_ix); int32 parent = ldg(parent_ids + initial_beam_ix); - for (int32 level = seq_len_b - 2; level >= 0; --level) { + bool found_bad = false; + for (int32 level = max_seq_len_b - 2; level >= 0; --level) { const int32 level_beam_ix = GET_IX(level, beam); const int32 level_parent_ix = GET_IX(level, parent); if (parent < 0 || parent > beam_width) { beams[level_beam_ix] = -1; parent = -1; + found_bad = true; } else { beams[level_beam_ix] = ldg(step_ids + level_parent_ix); parent = ldg(parent_ids + level_parent_ix); } } + // Not necessary when using a BeamSearchDecoder, but necessary + // when a user feeds in possibly broken trajectory (i.e., non-eos + // entries in a beam following eos entries). + if (!found_bad) { + bool finished = false; + for (int32 time = 0; time < max_seq_len_b; ++time) { + const int32 level_beam_ix = GET_IX(time, beam); + if (finished) { + beams[level_beam_ix] = end_token; + } else if (beams[level_beam_ix] == end_token) { + finished = true; + } + } + } #undef GET_IX } } @@ -62,20 +82,23 @@ struct GatherTree { void operator()(OpKernelContext* ctx, const GPUDevice& d, typename TTypes::ConstTensor step_ids, typename TTypes::ConstTensor parent_ids, - typename TTypes::ConstMatrix sequence_length, - typename TTypes::Tensor beams) { + TTypes::ConstVec max_sequence_length, + const T end_token, typename TTypes::Tensor beams) { const int32 max_time = parent_ids.dimension(0); const int32 batch_size = parent_ids.dimension(1); const int32 beam_width = parent_ids.dimension(2); - // First kernel launch to zero things out - beams.device(d) = beams.constant(T(-1)); + // First kernel launch to "zero" things out + beams.device(d) = beams.constant(end_token); CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * beam_width, d); // clang-format off GatherTreeOpKernel <<>>( batch_size, max_time, beam_width, - step_ids.data(), parent_ids.data(), sequence_length.data(), + step_ids.data(), + parent_ids.data(), + max_sequence_length.data(), + end_token, beams.data()); // clang-format on } diff --git a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc index 6c445cd4606381ed56d91000bc5e42d874ca0c5c..71539b6f592f0c8e53c4bb3801d1e35f34814966 100644 --- a/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc +++ b/tensorflow/contrib/seq2seq/ops/beam_search_ops.cc @@ -25,27 +25,27 @@ using shape_inference::ShapeHandle; REGISTER_OP("GatherTree") .Input("step_ids: T") .Input("parent_ids: T") - .Input("sequence_length: T") + .Input("max_sequence_lengths: int32") + .Input("end_token: T") .Output("beams: T") .Attr("T: {int32}") .SetShapeFn([](InferenceContext* c) { - ShapeHandle step_ids, parent_ids, sequence_length; + ShapeHandle step_ids, parent_ids, max_sequence_lengths, end_token; // step_ids, parent_ids, and output are all shaped: // [max_time, batch_size, beam_width]. - // sequence_length is shaped [batch_size, beam_width]. + // max_sequence_length is shaped [batch_size] and end_token is a scalar. TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 3, &step_ids)); TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &parent_ids)); - TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &sequence_length)); - - DimensionHandle batch_size = c->Dim(step_ids, 1); - DimensionHandle beam_width = c->Dim(step_ids, 2); - + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 1, &max_sequence_lengths)); + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &end_token)); TF_RETURN_IF_ERROR(c->Merge(step_ids, parent_ids, &step_ids)); + DimensionHandle batch_size = c->Dim(step_ids, 1); TF_RETURN_IF_ERROR( - c->Merge(batch_size, c->Dim(sequence_length, 0), &batch_size)); - TF_RETURN_IF_ERROR( - c->Merge(beam_width, c->Dim(sequence_length, 1), &beam_width)); + c->Merge(batch_size, c->Dim(max_sequence_lengths, 0), &batch_size)); + ShapeHandle step_ids_prefix = c->Matrix(c->Dim(step_ids, 0), batch_size); + TF_RETURN_IF_ERROR(c->MergePrefix(step_ids, step_ids_prefix, &step_ids, + &step_ids_prefix)); c->set_output(0, step_ids); return tensorflow::Status::OK(); @@ -53,15 +53,19 @@ REGISTER_OP("GatherTree") .Doc(R"doc( Calculates the full beams from the per-step ids and parent beam ids. -This op implements the following mathematical equations: +On CPU, if an out of bound parent id is found, an error is returned. +On GPU, if an out of bound parent id is found, a -1 is stored in the +corresponding output value and the execution for that beam returns early. + +For a given beam, past the time step containing the first decoded `end_token` +all values are filled in with `end_token`. -```python -TODO(ebrevdo): fill in -``` +TODO(ebrevdo): fill in the remainder of this docstring. step_ids: `[max_time, batch_size, beam_width]`. parent_ids: `[max_time, batch_size, beam_width]`. -sequence_length: `[batch_size, beam_width]`. +max_sequence_lengths: `[batch_size]`. +end_token: `[]`. beams: `[max_time, batch_size, beam_width]`. )doc"); diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py index 2caeb9eb614382c815984391df87a70516f519b2..d2beac5f31460ec1c0d978a9f6fcd0e0f09cb9b4 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py @@ -54,15 +54,18 @@ class TestGatherTree(test.TestCase): [[0, 0, 0], [1, 2, 0], [2, 1, 1]]], dtype=np.int32).transpose([1, 0, 2]) - # sequence_lengths is shaped (batch_size = 2, beam_width = 3) - sequence_lengths = [[3, 3, 3], [3, 3, 3]] + # sequence_lengths is shaped (batch_size = 3) + max_sequence_lengths = [3, 3] expected_result = np.array( [[[2, 2, 2], [6, 5, 6], [7, 8, 9]], [[2, 4, 4], [7, 6, 6], [8, 9, 10]]]).transpose([1, 0, 2]) res = beam_search_ops.gather_tree( - predicted_ids, parent_ids, sequence_lengths) + predicted_ids, + parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=11) with self.test_session() as sess: res_ = sess.run(res) @@ -80,8 +83,7 @@ class TestEosMasking(test.TestCase): ]) eos_token = 0 - previously_finished = constant_op.constant( - [[0, 1, 0], [0, 1, 1]], dtype=dtypes.float32) + previously_finished = np.array([[0, 1, 0], [0, 1, 1]], dtype=bool) masked = beam_search_decoder._mask_probs(probs, eos_token, previously_finished) diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py index 50cccf392fdac75f551b180987aff0b31da0893e..277c5b6ef76bce8d59e47cf0026c6e2b1d5cf1e2 100644 --- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py +++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_ops_test.py @@ -19,6 +19,8 @@ from __future__ import division from __future__ import print_function # pylint: enable=unused-import +import itertools + import numpy as np from tensorflow.contrib.seq2seq.python.ops import beam_search_ops @@ -34,31 +36,37 @@ class GatherTreeTest(test.TestCase): def testGatherTreeOne(self): # (max_time = 4, batch_size = 1, beams = 3) + end_token = 10 step_ids = _transpose_batch_time( [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]]]) - sequence_length = [[3, 3, 3]] - expected_result = _transpose_batch_time( - [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]]) + max_sequence_lengths = [3] + expected_result = _transpose_batch_time([[[2, 2, 2], [6, 5, 6], [7, 8, 9], + [10, 10, 10]]]) beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) + step_ids=step_ids, + parent_ids=parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=end_token) with self.test_session(use_gpu=True): self.assertAllEqual(expected_result, beams.eval()) def testBadParentValuesOnCPU(self): # (batch_size = 1, max_time = 4, beams = 3) # bad parent in beam 1 time 1 + end_token = 10 step_ids = _transpose_batch_time( [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) - sequence_length = [[3, 3, 3]] + max_sequence_lengths = [3] with ops.device("/cpu:0"): beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) + step_ids=step_ids, + parent_ids=parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=end_token) with self.test_session(): with self.assertRaisesOpError( r"parent id -1 at \(batch, time, beam\) == \(0, 0, 1\)"): @@ -71,82 +79,63 @@ class GatherTreeTest(test.TestCase): return # (max_time = 4, batch_size = 1, beams = 3) # bad parent in beam 1 time 1; appears as a negative index at time 0 + end_token = 10 step_ids = _transpose_batch_time( [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [-1, -1, -1]]]) parent_ids = _transpose_batch_time( [[[0, 0, 0], [0, -1, 1], [2, 1, 2], [-1, -1, -1]]]) - sequence_length = [[3, 3, 3]] - expected_result = _transpose_batch_time( - [[[2, -1, 2], [6, 5, 6], [7, 8, 9], [-1, -1, -1]]]) + max_sequence_lengths = [3] + expected_result = _transpose_batch_time([[[2, -1, 2], [6, 5, 6], [7, 8, 9], + [10, 10, 10]]]) with ops.device("/device:GPU:0"): beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) + step_ids=step_ids, + parent_ids=parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=end_token) with self.test_session(use_gpu=True): self.assertAllEqual(expected_result, beams.eval()) def testGatherTreeBatch(self): - # sequence_length is [batch_size, beam_width] = [4, 5] - sequence_length = [[0] * 5, [1] * 5, [2] * 5, [3] * 5] + batch_size = 10 + beam_width = 15 + max_time = 8 + max_sequence_lengths = [0, 1, 2, 4, 7, 8, 9, 10, 11, 0] + end_token = 5 with self.test_session(use_gpu=True): - # (max_time = 4, batch_size = 4, beam_width = 5) - step_ids = _transpose_batch_time( - [[[3, 4, 0, 4, 0], - [4, 2, 0, 3, 1], - [1, 1, 3, 2, 2], - [3, 1, 2, 3, 4]], - [[3, 4, 0, 4, 0], - [4, 2, 0, 3, 1], - [1, 1, 3, 2, 2], - [3, 1, 2, 3, 4]], - [[1, 2, 3, 4, 2], - [2, 1, 1, 3, 2], - [3, 0, 1, 0, 0], - [3, 4, 0, 2, 4]], - [[0, 2, 2, 3, 1], - [3, 2, 2, 2, 3], - [3, 4, 3, 0, 3], - [1, 2, 2, 2, 4]]]) - parent_ids = _transpose_batch_time( - [[[4, 2, 4, 3, 4], - [3, 4, 0, 2, 0], - [3, 1, 3, 2, 2], - [0, 2, 1, 4, 2]], - [[4, 2, 4, 3, 4], - [3, 4, 0, 2, 0], - [3, 1, 3, 2, 2], - [0, 2, 1, 4, 2]], - [[3, 0, 0, 4, 0], - [1, 2, 4, 2, 2], - [4, 4, 0, 3, 0], - [2, 4, 4, 3, 0]], - [[3, 1, 4, 1, 3], - [3, 2, 4, 0, 4], - [1, 0, 1, 4, 2], - [0, 3, 2, 0, 1]]]) - expected_beams = _transpose_batch_time( - [[[-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1]], - [[3, 4, 0, 4, 0], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1]], - [[2, 3, 2, 3, 3], - [2, 1, 1, 3, 2], - [-1, -1, -1, -1, -1], - [-1, -1, -1, -1, -1]], - [[2, 3, 2, 1, 1], - [2, 3, 2, 3, 2], - [3, 4, 3, 0, 3], - [-1, -1, -1, -1, -1]]]) + step_ids = np.random.randint( + 0, high=end_token + 1, size=(max_time, batch_size, beam_width)) + parent_ids = np.random.randint( + 0, high=beam_width - 1, size=(max_time, batch_size, beam_width)) beams = beam_search_ops.gather_tree( - step_ids=step_ids, parent_ids=parent_ids, - sequence_length=sequence_length) - self.assertAllEqual(expected_beams, beams.eval()) + step_ids=step_ids.astype(np.int32), + parent_ids=parent_ids.astype(np.int32), + max_sequence_lengths=max_sequence_lengths, + end_token=end_token) + + self.assertEqual((max_time, batch_size, beam_width), beams.shape) + beams_value = beams.eval() + for b in range(batch_size): + # Past max_sequence_lengths[b], we emit all end tokens. + b_value = beams_value[max_sequence_lengths[b]:, b, :] + self.assertAllClose(b_value, end_token * np.ones_like(b_value)) + for batch, beam in itertools.product( + range(batch_size), range(beam_width)): + v = np.squeeze(beams_value[:, batch, beam]) + if end_token in v: + found_bad = np.where(v == -1)[0] + self.assertEqual(0, len(found_bad)) + found = np.where(v == end_token)[0] + found = found[0] # First occurrence of end_token. + # If an end_token is found, everything before it should be a + # valid id and everything after it should be -1. + if found > 0: + self.assertAllEqual( + v[:found - 1] >= 0, np.ones_like(v[:found - 1], dtype=bool)) + self.assertAllClose(v[found + 1:], + end_token * np.ones_like(v[found + 1:])) if __name__ == "__main__": diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py index e22912ac5c9e378587d092ae2bed56929fe2a8e7..5be0c92243da10af438be97fab982515266be1de 100644 --- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py +++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py @@ -20,9 +20,10 @@ from __future__ import print_function import collections +import numpy as np + from tensorflow.contrib.seq2seq.python.ops import beam_search_ops from tensorflow.contrib.seq2seq.python.ops import decoder -from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -252,6 +253,20 @@ class BeamSearchDecoder(decoder.Decoder): output_shape_with_unknown_batch) return nest.map_structure(lambda s: s[1:], layer_output_shape) + @property + def tracks_own_finished(self): + """The BeamSearchDecoder shuffles its beams and their finished state. + + For this reason, it conflicts with the `dynamic_decode` function's + tracking of finished states. Setting this property to true avoids + early stopping of decoding due to mismanagement of the finished state + in `dynamic_decode`. + + Returns: + `True`. + """ + return True + @property def output_size(self): # Return the cell output and the id @@ -302,15 +317,23 @@ class BeamSearchDecoder(decoder.Decoder): output. sequence_lengths: An `int64` tensor shaped `[batch_size, beam_width]`. The sequence lengths determined for each beam during decode. + **NOTE** These are ignored; the updated sequence lengths are stored in + `final_state.lengths`. Returns: - outputs: An instance of FinalBeamSearchDecoderOutput where the + outputs: An instance of `FinalBeamSearchDecoderOutput` where the predicted_ids are the result of calling _gather_tree. - final_state: The same input instance of BeamSearchDecoderState. + final_state: The same input instance of `BeamSearchDecoderState`. """ + del sequence_lengths + # Get max_sequence_length across all beams for each batch. + max_sequence_lengths = math_ops.to_int32( + math_ops.reduce_max(final_state.lengths, axis=1)) predicted_ids = beam_search_ops.gather_tree( - outputs.predicted_ids, outputs.parent_ids, - sequence_length=sequence_lengths) + outputs.predicted_ids, + outputs.parent_ids, + max_sequence_lengths=max_sequence_lengths, + end_token=self._end_token) outputs = FinalBeamSearchDecoderOutput( beam_search_decoder_output=outputs, predicted_ids=predicted_ids) return outputs, final_state @@ -390,17 +413,17 @@ class BeamSearchDecoder(decoder.Decoder): We do this so that we can use nest and not run into problems with shapes. Args: - t: Tensor of dimension [batch_size*beam_width, s] - s: Tensor, Python int, or TensorShape. + t: `Tensor`, either scalar or shaped `[batch_size * beam_width] + s`. + s: `Tensor`, Python int, or `TensorShape`. Returns: - Either a reshaped version of t with dimension - [batch_size, beam_width, s] if t's first dimension is of size - batch_size*beam_width or t if not. + If `t` is a matrix or higher order tensor, then the return value is + `t` reshaped to `[batch_size, beam_width] + s`. Otherwise `t` is + returned unchanged. Raises: - TypeError: If t is an instance of TensorArray. - ValueError: If the rank of t is not statically known. + TypeError: If `t` is an instance of `TensorArray`. + ValueError: If the rank of `t` is not statically known. """ _check_maybe(t) if t.shape.ndims >= 1: @@ -411,19 +434,19 @@ class BeamSearchDecoder(decoder.Decoder): def _maybe_merge_batch_beams(self, t, s): """Splits the tensor from a batch by beams into a batch of beams. - More exactly, t is a tensor of dimension [batch_size*beam_width, s]. We - reshape this into [batch_size, beam_width, s] + More exactly, `t` is a tensor of dimension `[batch_size * beam_width] + s`, + then we reshape it to `[batch_size, beam_width] + s`. Args: - t: Tensor of dimension [batch_size*beam_width, s] - s: Tensor, Python int, or TensorShape. + t: `Tensor` of dimension `[batch_size * beam_width] + s`. + s: `Tensor`, Python int, or `TensorShape`. Returns: - A reshaped version of t with dimension [batch_size, beam_width, s]. + A reshaped version of t with shape `[batch_size, beam_width] + s`. Raises: - TypeError: If t is an instance of TensorArray. - ValueError: If the rank of t is not statically known. + TypeError: If `t` is an instance of `TensorArray`. + ValueError: If the rank of `t` is not statically known. """ _check_maybe(t) if t.shape.ndims >= 2: @@ -521,14 +544,12 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size, # Calculate the continuation lengths by adding to all continuing beams. vocab_size = logits.shape[-1].value or array_ops.shape(logits)[-1] lengths_to_add = array_ops.one_hot( - indices=array_ops.tile( - array_ops.reshape(end_token, [1, 1]), [batch_size, beam_width]), + indices=array_ops.fill([batch_size, beam_width], end_token), depth=vocab_size, - on_value=constant_op.constant(0, dtype=dtypes.int64), - off_value=constant_op.constant(1, dtype=dtypes.int64), + on_value=np.int64(0), off_value=np.int64(1), dtype=dtypes.int64) - add_mask = (1 - math_ops.to_int64(previously_finished)) - lengths_to_add = array_ops.expand_dims(add_mask, 2) * lengths_to_add + add_mask = math_ops.to_int64(math_ops.logical_not(previously_finished)) + lengths_to_add *= array_ops.expand_dims(add_mask, 2) new_prediction_lengths = ( lengths_to_add + array_ops.expand_dims(prediction_lengths, 2)) @@ -589,12 +610,11 @@ def _beam_search_step(time, logits, next_cell_state, beam_state, batch_size, name="next_beam_finished") # Calculate the length of the next predictions. - # 1. Finished beams remain unchanged - # 2. Beams that are now finished (EOS predicted) remain unchanged - # 3. Beams that are not yet finished have their length increased by 1 - lengths_to_add = math_ops.to_int64( - math_ops.not_equal(next_word_ids, end_token)) - lengths_to_add = (1 - math_ops.to_int64(next_finished)) * lengths_to_add + # 1. Finished beams remain unchanged. + # 2. Beams that are now finished (EOS predicted) have their length + # increased by 1. + # 3. Beams that are not yet finished have their length increased by 1. + lengths_to_add = math_ops.to_int64(math_ops.logical_not(previously_finished)) next_prediction_len = _tensor_gather_helper( gather_indices=next_beam_ids, gather_from=beam_state.lengths, @@ -652,13 +672,20 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight): def _length_penalty(sequence_lengths, penalty_factor): """Calculates the length penalty. See https://arxiv.org/abs/1609.08144. + Returns the length penalty tensor: + ``` + [(5+sequence_lengths)/6]**penalty_factor + ``` + where all operations are performed element-wise. + Args: - sequence_lengths: The sequence length of all hypotheses, a tensor - of shape [beam_size, vocab_size]. + sequence_lengths: `Tensor`, the sequence lengths of each hypotheses. penalty_factor: A scalar that weights the length penalty. Returns: - The length penalty factor, a tensor fo shape [beam_size]. + If the penalty is `0`, returns the scalar `1.0`. Otherwise returns + the length penalty factor, a tensor with the same shape as + `sequence_lengths`. """ penalty_factor = ops.convert_to_tensor(penalty_factor, name="penalty_factor") penalty_factor.set_shape(()) # penalty should be a scalar. @@ -680,8 +707,7 @@ def _mask_probs(probs, eos_token, finished): eos_token: An int32 id corresponding to the EOS token to allocate probability to. finished: A boolean tensor of shape `[batch_size, beam_width]` that - specifies which - elements in the beam are finished already. + specifies which elements in the beam are finished already. Returns: A tensor of shape `[batch_size, beam_width, vocab_size]`, where unfinished @@ -689,10 +715,6 @@ def _mask_probs(probs, eos_token, finished): probability on the EOS token. """ vocab_size = array_ops.shape(probs)[2] - finished_mask = array_ops.expand_dims( - math_ops.to_float(1. - math_ops.to_float(finished)), 2) - # These examples are not finished and we leave them - non_finished_examples = finished_mask * probs # All finished examples are replaced with a vector that has all # probability on EOS finished_row = array_ops.one_hot( @@ -701,8 +723,13 @@ def _mask_probs(probs, eos_token, finished): dtype=probs.dtype, on_value=0., off_value=probs.dtype.min) - finished_examples = (1. - finished_mask) * finished_row - return finished_examples + non_finished_examples + finished_probs = array_ops.tile( + array_ops.reshape(finished_row, [1, 1, -1]), + array_ops.concat([array_ops.shape(finished), [1]], 0)) + finished_mask = array_ops.tile( + array_ops.expand_dims(finished, 2), [1, 1, vocab_size]) + + return array_ops.where(finished_mask, finished_probs, probs) def _maybe_tensor_gather_helper(gather_indices, gather_from, batch_size, diff --git a/tensorflow/contrib/seq2seq/python/ops/decoder.py b/tensorflow/contrib/seq2seq/python/ops/decoder.py index fbe53fc60ada85c40970870c6d0bdb93d17ea6d4..f14974b9d5ca8cbcfd9f91086ca0a90ceff48f43 100644 --- a/tensorflow/contrib/seq2seq/python/ops/decoder.py +++ b/tensorflow/contrib/seq2seq/python/ops/decoder.py @@ -100,16 +100,36 @@ class Decoder(object): Returns: `(outputs, next_state, next_inputs, finished)`: `outputs` is an object - containing the decoder output, `next_state` is a (structure of) state tensors - and TensorArrays, `next_inputs` is the tensor that should be used as input for - the next step, `finished` is a boolean tensor telling whether the sequence - is complete, for each sequence in the batch. + containing the decoder output, `next_state` is a (structure of) state + tensors and TensorArrays, `next_inputs` is the tensor that should be used + as input for the next step, `finished` is a boolean tensor telling whether + the sequence is complete, for each sequence in the batch. """ raise NotImplementedError def finalize(self, outputs, final_state, sequence_lengths): raise NotImplementedError + @property + def tracks_own_finished(self): + """Describes whether the Decoder keeps track of finished states. + + Most decoders will emit a true/false `finished` value independently + at each time step. In this case, the `dynamic_decode` function keeps track + of which batch entries are already finished, and performs a logical OR to + insert new batches to the finished set. + + Some decoders, however, shuffle batches / beams between time steps and + `dynamic_decode` will mix up the finished state across these entries because + it does not track the reshuffle across time steps. In this case, it is + up to the decoder to declare that it will keep track of its own finished + state by setting this property to `True`. + + Returns: + Python bool. + """ + return False + def _create_zero_outputs(size, dtype, batch_size): """Create a zero outputs Tensor structure.""" @@ -232,7 +252,10 @@ def dynamic_decode(decoder, """ (next_outputs, decoder_state, next_inputs, decoder_finished) = decoder.step(time, inputs, state) - next_finished = math_ops.logical_or(decoder_finished, finished) + if decoder.tracks_own_finished: + next_finished = decoder_finished + else: + next_finished = math_ops.logical_or(decoder_finished, finished) if maximum_iterations is not None: next_finished = math_ops.logical_or( next_finished, time + 1 >= maximum_iterations) diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD index 8a1c9ba0a2ca01396bec662214f9a5f0d732f34b..67011c8fef6c4f54db2626ffe7ae1299bddbb352 100644 --- a/tensorflow/contrib/session_bundle/BUILD +++ b/tensorflow/contrib/session_bundle/BUILD @@ -136,7 +136,6 @@ py_test( ":gc", ":manifest_proto_py", "//tensorflow/core:protos_all_py", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", @@ -411,8 +410,6 @@ tf_cc_test( ":test_util", "//tensorflow/cc/saved_model:signature_constants", "//tensorflow/cc/saved_model:tag_constants", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core:test", diff --git a/tensorflow/contrib/signal/BUILD b/tensorflow/contrib/signal/BUILD index 2204b684ac993cd82e69b3fd74801bff610b5fd4..b67090dd509f321c8d28436fa135fb871aee976d 100644 --- a/tensorflow/contrib/signal/BUILD +++ b/tensorflow/contrib/signal/BUILD @@ -32,8 +32,8 @@ py_library( srcs_version = "PY2AND3", deps = [ "//tensorflow/core:protos_all_py", - "//tensorflow/python:framework", "//tensorflow/python:tf_optimizer", + "//tensorflow/python:training", ], ) diff --git a/tensorflow/contrib/slim/BUILD b/tensorflow/contrib/slim/BUILD index d2664b612cdbcae3a346b68e9caee654c48a69cd..23c23af2f4815c3b1d75eb955b9026dfb9b00194 100644 --- a/tensorflow/contrib/slim/BUILD +++ b/tensorflow/contrib/slim/BUILD @@ -48,7 +48,6 @@ py_library( srcs = ["python/slim/learning.py"], srcs_version = "PY2AND3", deps = [ - "//tensorflow/contrib/framework:framework_py", "//tensorflow/contrib/training:training_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:client", @@ -78,7 +77,6 @@ py_test( "//tensorflow/contrib/losses:losses_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD index fc71a5fe415d4d34bd38e43bf33cefffcddaea6f..5daabbd62e7e63608a7a86a8b7fb0bc0d570b28b 100644 --- a/tensorflow/contrib/slim/python/slim/data/BUILD +++ b/tensorflow/contrib/slim/python/slim/data/BUILD @@ -68,13 +68,13 @@ py_test( ":tfexample_decoder", "//tensorflow/contrib/slim:queues", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:image_ops", "//tensorflow/python:io_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", + "//tensorflow/python:session", ], ) @@ -187,6 +187,7 @@ py_test( "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:image_ops", + "//tensorflow/python:lookup_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", "//third_party/py/numpy", diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py index 094568389cfdd2fd83b939cb9242694391f3844b..0544404e9e252cca6d3650b805b91be25d705eea 100644 --- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py +++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py @@ -207,6 +207,76 @@ class Tensor(ItemHandler): return tensor +class LookupTensor(Tensor): + """An ItemHandler that returns a parsed Tensor, the result of a lookup.""" + + def __init__(self, + tensor_key, + table, + shape_keys=None, + shape=None, + default_value=''): + """Initializes the LookupTensor handler. + + See Tensor. Simply calls a vocabulary (most often, a label mapping) lookup. + + Args: + tensor_key: the name of the `TFExample` feature to read the tensor from. + table: A tf.lookup table. + shape_keys: Optional name or list of names of the TF-Example feature in + which the tensor shape is stored. If a list, then each corresponds to + one dimension of the shape. + shape: Optional output shape of the `Tensor`. If provided, the `Tensor` is + reshaped accordingly. + default_value: The value used when the `tensor_key` is not found in a + particular `TFExample`. + + Raises: + ValueError: if both `shape_keys` and `shape` are specified. + """ + self._table = table + super(LookupTensor, self).__init__(tensor_key, shape_keys, shape, + default_value) + + def tensors_to_item(self, keys_to_tensors): + unmapped_tensor = super(LookupTensor, self).tensors_to_item(keys_to_tensors) + return self._table.lookup(unmapped_tensor) + + +class BackupHandler(ItemHandler): + """An ItemHandler that tries two ItemHandlers in order.""" + + def __init__(self, handler, backup): + """Initializes the BackupHandler handler. + + If the first Handler's tensors_to_item returns a Tensor with no elements, + the second Handler is used. + + Args: + handler: The primary ItemHandler. + backup: The backup ItemHandler. + + Raises: + ValueError: if either is not an ItemHandler. + """ + if not isinstance(handler, ItemHandler): + raise ValueError('Primary handler is of type %s instead of ItemHandler' + % type(handler)) + if not isinstance(backup, ItemHandler): + raise ValueError('Backup handler is of type %s instead of ItemHandler' + % type(backup)) + self._handler = handler + self._backup = backup + super(BackupHandler, self).__init__(handler.keys + backup.keys) + + def tensors_to_item(self, keys_to_tensors): + item = self._handler.tensors_to_item(keys_to_tensors) + return control_flow_ops.cond( + pred=math_ops.equal(math_ops.reduce_prod(array_ops.shape(item)), 0), + true_fn=lambda: self._backup.tensors_to_item(keys_to_tensors), + false_fn=lambda: item) + + class SparseTensor(ItemHandler): """An ItemHandler for SparseTensors.""" diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py index 60d1eba07fbe1ab9ff823478c8fadb3066c866e7..d783d4fef42bb2acffe7eb8b155c5efaed7896d9 100644 --- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py +++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import image_ops +from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test @@ -811,6 +812,87 @@ class TFExampleDecoderTest(test.TestCase): self.assertAllEqual(np.squeeze(output_image[0, :, :, :]), image) self.assertAllEqual(np.squeeze(output_image[1, :, :, :]), image) + def testDecodeExampleWithLookup(self): + + example = example_pb2.Example(features=feature_pb2.Features(feature={ + 'image/object/class/text': self._BytesFeature( + np.array(['cat', 'dog', 'guinea pig'])), + })) + serialized_example = example.SerializeToString() + # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2 + table = lookup_ops.index_table_from_tensor( + constant_op.constant(['dog', 'guinea pig', 'cat'])) + + with self.test_session() as sess: + sess.run(lookup_ops.tables_initializer()) + + serialized_example = array_ops.reshape(serialized_example, shape=[]) + + keys_to_features = { + 'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string), + } + + items_to_handlers = { + 'labels': + tfexample_decoder.LookupTensor('image/object/class/text', table), + } + + decoder = tfexample_decoder.TFExampleDecoder(keys_to_features, + items_to_handlers) + obtained_class_ids = decoder.decode(serialized_example)[0].eval() + + self.assertAllClose([2, 0, 1], obtained_class_ids) + + def testDecodeExampleWithBackupHandlerLookup(self): + + example1 = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'image/object/class/text': + self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])), + 'image/object/class/label': + self._EncodedInt64Feature(np.array([42, 10, 900])) + })) + example2 = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'image/object/class/text': + self._BytesFeature(np.array(['cat', 'dog', 'guinea pig'])), + })) + example3 = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'image/object/class/label': + self._EncodedInt64Feature(np.array([42, 10, 901])) + })) + # 'dog' -> 0, 'guinea pig' -> 1, 'cat' -> 2 + table = lookup_ops.index_table_from_tensor( + constant_op.constant(['dog', 'guinea pig', 'cat'])) + keys_to_features = { + 'image/object/class/text': parsing_ops.VarLenFeature(dtypes.string), + 'image/object/class/label': parsing_ops.VarLenFeature(dtypes.int64), + } + backup_handler = tfexample_decoder.BackupHandler( + handler=tfexample_decoder.Tensor('image/object/class/label'), + backup=tfexample_decoder.LookupTensor('image/object/class/text', table)) + items_to_handlers = { + 'labels': backup_handler, + } + decoder = tfexample_decoder.TFExampleDecoder(keys_to_features, + items_to_handlers) + obtained_class_ids_each_example = [] + with self.test_session() as sess: + sess.run(lookup_ops.tables_initializer()) + for example in [example1, example2, example3]: + serialized_example = array_ops.reshape( + example.SerializeToString(), shape=[]) + obtained_class_ids_each_example.append( + decoder.decode(serialized_example)[0].eval()) + + self.assertAllClose([42, 10, 900], obtained_class_ids_each_example[0]) + self.assertAllClose([2, 0, 1], obtained_class_ids_each_example[1]) + self.assertAllClose([42, 10, 901], obtained_class_ids_each_example[2]) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/slim/python/slim/learning.py b/tensorflow/contrib/slim/python/slim/learning.py index 5ee014a1f11a6b0d11857d209f27b134b737275d..def00b76184ba4e1fc630cd83d8e055448100562 100644 --- a/tensorflow/contrib/slim/python/slim/learning.py +++ b/tensorflow/contrib/slim/python/slim/learning.py @@ -552,7 +552,8 @@ def train(train_op, sync_optimizer=None, session_config=None, session_wrapper=None, - trace_every_n_steps=None): + trace_every_n_steps=None, + ignore_live_threads=False): """Runs a training loop using a TensorFlow supervisor. When the sync_optimizer is supplied, gradient updates are applied @@ -615,6 +616,9 @@ def train(train_op, trace_every_n_steps: produce and save a `Timeline` in Chrome trace format and add it to the summaries every `trace_every_n_steps`. If None, no trace information will be produced or saved. + ignore_live_threads: If `True` ignores threads that remain running after + a grace period when stopping the supervisor, instead of raising a + RuntimeError. Returns: the value of the loss function after training. @@ -772,7 +776,10 @@ def train(train_op, if logdir and sv.is_chief: logging.info('Finished training! Saving model to disk.') sv.saver.save(sess, sv.save_path, global_step=sv.global_step) - sv.stop(threads, close_summary_writer=True) + sv.stop( + threads, + close_summary_writer=True, + ignore_live_threads=ignore_live_threads) except errors.AbortedError: # Always re-run on AbortedError as it indicates a restart of one of the diff --git a/tensorflow/contrib/stateless/BUILD b/tensorflow/contrib/stateless/BUILD index 865fb72a55b9a83b8354a100af843abaefc79980..6e259e1d32be64f3b593faf73e8af4f704d72349 100644 --- a/tensorflow/contrib/stateless/BUILD +++ b/tensorflow/contrib/stateless/BUILD @@ -21,7 +21,7 @@ py_library( srcs_version = "PY2AND3", deps = [ ":stateless_random_ops", - "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py index 9a36bdc2f9558220fa6cc47d5bb95d6e49a480f7..cd4d46aa07bfa92b8243f2f168fd1e4682ad70e2 100644 --- a/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py +++ b/tensorflow/contrib/stateless/python/kernel_tests/stateless_random_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import numpy as np from tensorflow.contrib import stateless +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import random_seed from tensorflow.python.ops import array_ops @@ -79,6 +80,21 @@ class StatelessOpsTest(test.TestCase): for s1, v1 in values: self.assertEqual(s0 == s1, np.all(v0 == v1)) + def testShapeType(self): + with self.test_session(use_gpu=True): + for shape_dtype in [dtypes.int32, dtypes.int64]: + seed_t = array_ops.placeholder(dtypes.int64, shape=[2]) + seeds = [(x, y) for x in range(5) for y in range(5)] * 3 + for stateless_op, _ in CASES: + for shape in (), (3,), (2, 5): + pure = stateless_op(constant_op.constant(shape, dtype=shape_dtype), + seed=seed_t) + values = [(seed, pure.eval(feed_dict={seed_t: seed})) + for seed in seeds] + for s0, v0 in values: + for s1, v1 in values: + self.assertEqual(s0 == s1, np.all(v0 == v1)) + if __name__ == '__main__': test.main() diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD index d09ad48e10a0dfe37860d302567f6cc241135422..da23f1c3806be73d43e44bf4b4079d81b2d61c8f 100644 --- a/tensorflow/contrib/summary/BUILD +++ b/tensorflow/contrib/summary/BUILD @@ -25,9 +25,9 @@ py_test( srcs_version = "PY2AND3", deps = [ ":summary_ops", - "//tensorflow/core:protos_all_py", + ":summary_test_util", + "//tensorflow/python:errors", "//tensorflow/python:framework_test_lib", - "//tensorflow/python:lib", "//tensorflow/python:platform", "//tensorflow/python:training", "//tensorflow/python/eager:function", @@ -42,16 +42,29 @@ py_library( visibility = ["//tensorflow:internal"], deps = [ ":gen_summary_ops", + "//tensorflow/python:array_ops", "//tensorflow/python:constant_op", - "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:layers_base", + "//tensorflow/python:resource_variable_ops", "//tensorflow/python:summary_op_util", "//tensorflow/python:training", + "//tensorflow/python:util", "//tensorflow/python/eager:context", ], ) +py_library( + name = "summary", + srcs = ["summary.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + ":summary_ops", + ], +) + filegroup( name = "all_files", srcs = glob( @@ -63,3 +76,17 @@ filegroup( ), visibility = ["//tensorflow:__subpackages__"], ) + +# NOTE: target cannot be testonly because it needs to be in the pip +# package. Sigh. +py_library( + name = "summary_test_util", + srcs = ["summary_test_util.py"], + srcs_version = "PY2AND3", + visibility = ["//tensorflow:internal"], + deps = [ + "//tensorflow/core:protos_all_py", + "//tensorflow/python:lib", + "//tensorflow/python:platform", + ], +) diff --git a/tensorflow/contrib/summary/summary.py b/tensorflow/contrib/summary/summary.py new file mode 100644 index 0000000000000000000000000000000000000000..ca82ea094c41c15f376e6f6f448b770c5cf291d7 --- /dev/null +++ b/tensorflow/contrib/summary/summary.py @@ -0,0 +1,40 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Contrib summary package. + +The operations in this package are safe to use with eager execution turned or on +off. + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# pylint: disable=unused-import +from tensorflow.contrib.summary.summary_ops import all_summary_ops +from tensorflow.contrib.summary.summary_ops import always_record_summaries +from tensorflow.contrib.summary.summary_ops import audio +from tensorflow.contrib.summary.summary_ops import create_summary_file_writer +from tensorflow.contrib.summary.summary_ops import eval_dir +from tensorflow.contrib.summary.summary_ops import generic +from tensorflow.contrib.summary.summary_ops import histogram +from tensorflow.contrib.summary.summary_ops import image +from tensorflow.contrib.summary.summary_ops import never_record_summaries +from tensorflow.contrib.summary.summary_ops import record_summaries_every_n_global_steps +from tensorflow.contrib.summary.summary_ops import scalar +from tensorflow.contrib.summary.summary_ops import should_record_summaries +from tensorflow.contrib.summary.summary_ops import summary_writer_initializer_op diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py index c8d0c14e1951a7c29eed096d2a2e9849c4326245..1d1c88944aba7b84f8b56d466c0532c938f90006 100644 --- a/tensorflow/contrib/summary/summary_ops.py +++ b/tensorflow/contrib/summary/summary_ops.py @@ -19,26 +19,35 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import os + from tensorflow.contrib.summary import gen_summary_ops from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.layers import utils +from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import summary_op_util from tensorflow.python.training import training_util - +from tensorflow.python.util import tf_contextlib # Name for a collection which is expected to have at most a single boolean # Tensor. If this tensor is True the summary ops will record summaries. _SHOULD_RECORD_SUMMARIES_NAME = "ShouldRecordSummaries" +_SUMMARY_COLLECTION_NAME = "_SUMMARY_V2" +_SUMMARY_WRITER_INIT_COLLECTION_NAME = "_SUMMARY_WRITER_V2" + def should_record_summaries(): """Returns boolean Tensor which is true if summaries should be recorded.""" should_record_collection = ops.get_collection(_SHOULD_RECORD_SUMMARIES_NAME) if not should_record_collection: - return constant_op.constant(False) + return False if len(should_record_collection) != 1: raise ValueError( "More than one tensor specified for whether summaries " @@ -47,22 +56,62 @@ def should_record_summaries(): # TODO(apassos) consider how to handle local step here. +@tf_contextlib.contextmanager def record_summaries_every_n_global_steps(n): """Sets the should_record_summaries Tensor to true if global_step % n == 0.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) - collection_ref[:] = [training_util.get_global_step() % n == 0] + old = collection_ref[:] + with ops.device("cpu:0"): + collection_ref[:] = [math_ops.equal(training_util.get_global_step() % n, 0)] + yield + collection_ref[:] = old +@tf_contextlib.contextmanager def always_record_summaries(): """Sets the should_record_summaries Tensor to always true.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) - collection_ref[:] = [constant_op.constant(True)] + old = collection_ref[:] + collection_ref[:] = [True] + yield + collection_ref[:] = old +@tf_contextlib.contextmanager def never_record_summaries(): """Sets the should_record_summaries Tensor to always false.""" collection_ref = ops.get_collection_ref(_SHOULD_RECORD_SUMMARIES_NAME) - collection_ref[:] = [constant_op.constant(False)] + old = collection_ref[:] + collection_ref[:] = [False] + yield + collection_ref[:] = old + + +class SummaryWriter(object): + """Encapsulates a summary writer.""" + + def __init__(self, resource): + self._resource = resource + if context.in_eager_mode(): + self._resource_deleter = resource_variable_ops.EagerResourceDeleter( + handle=self._resource, handle_device="cpu:0") + + def set_as_default(self): + context.context().summary_writer_resource = self._resource + + @tf_contextlib.contextmanager + def as_default(self): + if self._resource is None: + yield + else: + old = context.context().summary_writer_resource + context.context().summary_writer_resource = self._resource + yield + # Flushes the summary writer in eager mode or in graph functions, but not + # in legacy graph mode (you're on your own there). + with ops.device("cpu:0"): + gen_summary_ops.flush_summary_writer(self._resource) + context.context().summary_writer_resource = old def create_summary_file_writer(logdir, @@ -70,22 +119,62 @@ def create_summary_file_writer(logdir, flush_secs=None, filename_suffix=None, name=None): - """Creates a summary file writer in the current context.""" - if max_queue is None: - max_queue = constant_op.constant(10) - if flush_secs is None: - flush_secs = constant_op.constant(120) - if filename_suffix is None: - filename_suffix = constant_op.constant("") - resource = gen_summary_ops.summary_writer(shared_name=name) - gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, - flush_secs, filename_suffix) - context.context().summary_writer_resource = resource + """Creates a summary file writer in the current context. + + Args: + logdir: a string, or None. If a string, creates a summary file writer + which writes to the directory named by the string. If None, returns + a mock object which acts like a summary writer but does nothing, + useful to use as a context manager. + max_queue: the largest number of summaries to keep in a queue; will + flush once the queue gets bigger than this. + flush_secs: the largest interval (in seconds) between flushes. + filename_suffix: optional suffix for the event file name. + name: name for the summary writer. + + Returns: + Either a summary writer or an empty object which can be used as a + summary writer. + """ + if logdir is None: + return SummaryWriter(None) + with ops.device("cpu:0"): + if max_queue is None: + max_queue = constant_op.constant(10) + if flush_secs is None: + flush_secs = constant_op.constant(120) + if filename_suffix is None: + filename_suffix = constant_op.constant("") + resource = gen_summary_ops.summary_writer(shared_name=name) + # TODO(apassos) ensure the initialization op runs when in graph mode; + # consider calling session.run here. + ops.add_to_collection( + _SUMMARY_WRITER_INIT_COLLECTION_NAME, + gen_summary_ops.create_summary_file_writer(resource, logdir, max_queue, + flush_secs, filename_suffix)) + return SummaryWriter(resource) def _nothing(): """Convenient else branch for when summaries do not record.""" - return False + return constant_op.constant(False) + + +def all_summary_ops(): + """Graph-mode only. Returns all summary ops.""" + if context.in_eager_mode(): + raise RuntimeError( + "tf.contrib.summary.all_summary_ops is only supported in graph mode.") + return ops.get_collection(_SUMMARY_COLLECTION_NAME) + + +def summary_writer_initializer_op(): + """Graph-mode only. Returns the list of ops to create all summary writers.""" + if context.in_eager_mode(): + raise RuntimeError( + "tf.contrib.summary.summary_writer_initializer_op is only " + "supported in graph mode.") + return ops.get_collection(_SUMMARY_WRITER_INIT_COLLECTION_NAME) def summary_writer_function(name, tensor, function, family=None): @@ -103,20 +192,27 @@ def summary_writer_function(name, tensor, function, family=None): def record(): with summary_op_util.summary_scope( name, family, values=[tensor]) as (tag, scope): - function(tag, scope) - return True + with ops.control_dependencies([function(tag, scope)]): + return constant_op.constant(True) - return control_flow_ops.cond( - should_record_summaries(), record, _nothing, name="") + if context.context().summary_writer_resource is None: + return control_flow_ops.no_op() + with ops.device("cpu:0"): + op = utils.smart_cond( + should_record_summaries(), record, _nothing, name="") + ops.add_to_collection(_SUMMARY_COLLECTION_NAME, op) + return op def generic(name, tensor, metadata, family=None): """Writes a tensor summary if possible.""" def function(tag, scope): - gen_summary_ops.write_summary(context.context().summary_writer_resource, - training_util.get_global_step(), tensor, - tag, metadata, name=scope) + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_summary( + context.context().summary_writer_resource, + training_util.get_global_step(), array_ops.identity(tensor), + tag, metadata, name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -124,9 +220,11 @@ def scalar(name, tensor, family=None): """Writes a scalar summary if possible.""" def function(tag, scope): - gen_summary_ops.write_scalar_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_scalar_summary( context.context().summary_writer_resource, - training_util.get_global_step(), tag, tensor, name=scope) + training_util.get_global_step(), tag, array_ops.identity(tensor), + name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -135,9 +233,11 @@ def histogram(name, tensor, family=None): """Writes a histogram summary if possible.""" def function(tag, scope): - gen_summary_ops.write_histogram_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_histogram_summary( context.context().summary_writer_resource, - training_util.get_global_step(), tag, tensor, name=scope) + training_util.get_global_step(), tag, array_ops.identity(tensor), + name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -148,10 +248,12 @@ def image(name, tensor, bad_color=None, max_images=3, family=None): def function(tag, scope): if bad_color is None: bad_color_ = constant_op.constant([255, 0, 0, 255], dtype=dtypes.uint8) - gen_summary_ops.write_image_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_image_summary( context.context().summary_writer_resource, - training_util.get_global_step(), tag, tensor, bad_color_, max_images, - name=scope) + training_util.get_global_step(), tag, array_ops.identity(tensor), + bad_color_, + max_images, name=scope) return summary_writer_function(name, tensor, function, family=family) @@ -160,13 +262,19 @@ def audio(name, tensor, sample_rate, max_outputs, family=None): """Writes an audio summary if possible.""" def function(tag, scope): - gen_summary_ops.write_audio_summary( + # Note the identity to move the tensor to the CPU. + return gen_summary_ops.write_audio_summary( context.context().summary_writer_resource, training_util.get_global_step(), tag, - tensor, + array_ops.identity(tensor), sample_rate=sample_rate, max_outputs=max_outputs, name=scope) return summary_writer_function(name, tensor, function, family=family) + + +def eval_dir(model_dir, name=None): + """Construct a logdir for an eval summary writer.""" + return os.path.join(model_dir, "eval" if not name else "eval_" + name) diff --git a/tensorflow/contrib/summary/summary_ops_test.py b/tensorflow/contrib/summary/summary_ops_test.py index 6958ee8dd83600d130293322c8680b3c0c0c02b2..de7ae6ec277a97235617882a7cc7e469eaebe26c 100644 --- a/tensorflow/contrib/summary/summary_ops_test.py +++ b/tensorflow/contrib/summary/summary_ops_test.py @@ -17,16 +17,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import tempfile from tensorflow.contrib.summary import summary_ops -from tensorflow.core.util import event_pb2 +from tensorflow.contrib.summary import summary_test_util from tensorflow.python.eager import function from tensorflow.python.eager import test from tensorflow.python.framework import errors from tensorflow.python.framework import test_util -from tensorflow.python.lib.io import tf_record from tensorflow.python.platform import gfile from tensorflow.python.training import training_util @@ -40,61 +38,53 @@ class TargetTest(test_util.TensorFlowTestCase): summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t0') def testShouldRecordSummary(self): - self.assertFalse(summary_ops.should_record_summaries().numpy()) - summary_ops.always_record_summaries() - self.assertTrue(summary_ops.should_record_summaries().numpy()) + self.assertFalse(summary_ops.should_record_summaries()) + with summary_ops.always_record_summaries(): + self.assertTrue(summary_ops.should_record_summaries()) def testSummaryOps(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() - summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t0') - summary_ops.always_record_summaries() - summary_ops.generic('tensor', 1, '') - summary_ops.scalar('scalar', 2.0) - summary_ops.histogram('histogram', [1.0]) - summary_ops.image('image', [[[[1.0]]]]) - summary_ops.audio('audio', [[1.0]], 1.0, 1) - # The working condition of the ops is tested in the C++ test so we just - # test here that we're calling them correctly. - self.assertTrue(gfile.Exists(logdir)) + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name='t0').as_default(), summary_ops.always_record_summaries(): + summary_ops.generic('tensor', 1, '') + summary_ops.scalar('scalar', 2.0) + summary_ops.histogram('histogram', [1.0]) + summary_ops.image('image', [[[[1.0]]]]) + summary_ops.audio('audio', [[1.0]], 1.0, 1) + # The working condition of the ops is tested in the C++ test so we just + # test here that we're calling them correctly. + self.assertTrue(gfile.Exists(logdir)) def testDefunSummarys(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() - summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t1') - summary_ops.always_record_summaries() - - @function.defun - def write(): - summary_ops.scalar('scalar', 2.0) + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name='t1').as_default(), summary_ops.always_record_summaries(): - write() + @function.defun + def write(): + summary_ops.scalar('scalar', 2.0) - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list(tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].simple_value, 2.0) + write() + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].simple_value, 2.0) def testSummaryName(self): training_util.get_or_create_global_step() logdir = tempfile.mkdtemp() - summary_ops.create_summary_file_writer(logdir, max_queue=0, name='t2') - summary_ops.always_record_summaries() - - summary_ops.scalar('scalar', 2.0) - - self.assertTrue(gfile.Exists(logdir)) - files = gfile.ListDirectory(logdir) - self.assertEqual(len(files), 1) - records = list(tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) - self.assertEqual(len(records), 2) - event = event_pb2.Event() - event.ParseFromString(records[1]) - self.assertEqual(event.summary.value[0].tag, 'scalar') + with summary_ops.create_summary_file_writer( + logdir, max_queue=0, + name='t2').as_default(), summary_ops.always_record_summaries(): + + summary_ops.scalar('scalar', 2.0) + + events = summary_test_util.events_from_file(logdir) + self.assertEqual(len(events), 2) + self.assertEqual(events[1].summary.value[0].tag, 'scalar') if __name__ == '__main__': diff --git a/tensorflow/contrib/summary/summary_test_util.py b/tensorflow/contrib/summary/summary_test_util.py new file mode 100644 index 0000000000000000000000000000000000000000..37b546d3ab3220f934ea3bf7ef8f5fe6ab29f683 --- /dev/null +++ b/tensorflow/contrib/summary/summary_test_util.py @@ -0,0 +1,41 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Utilities to test summaries.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +from tensorflow.core.util import event_pb2 +from tensorflow.python.lib.io import tf_record +from tensorflow.python.platform import gfile + + +def events_from_file(logdir): + """Returns all events in the single eventfile in logdir.""" + assert gfile.Exists(logdir) + files = gfile.ListDirectory(logdir) + assert len(files) == 1, "Found more than one file in logdir: %s" % files + records = list( + tf_record.tf_record_iterator(os.path.join(logdir, files[0]))) + result = [] + for r in records: + event = event_pb2.Event() + event.ParseFromString(r) + result.append(event) + return result diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/contrib/tensorboard/db/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..d8bbf87d2cecaec9b612e45e82295cebd3ac4c7f --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/BUILD @@ -0,0 +1,62 @@ +# Description: +# TensorBoard database code. + +package(default_visibility = ["//tensorflow:internal"]) + +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "tf_cc_test") + +cc_library( + name = "schema", + srcs = ["schema.cc"], + hdrs = ["schema.h"], + deps = [ + "//tensorflow/core:lib", + "//tensorflow/core/lib/db:sqlite", + ], +) + +tf_cc_test( + name = "schema_test", + srcs = ["schema_test.cc"], + deps = [ + ":schema", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/db:sqlite", + ], +) + +cc_library( + name = "summary_db_writer", + srcs = ["summary_db_writer.cc"], + hdrs = ["summary_db_writer.h"], + deps = [ + ":schema", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/kernels:summary_interface", + "//tensorflow/core/lib/db:sqlite", + ], +) + +tf_cc_test( + name = "summary_db_writer_test", + srcs = ["summary_db_writer_test.cc"], + deps = [ + ":summary_db_writer", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/lib/db:sqlite", + ], +) + +filegroup( + name = "all_files", + srcs = glob(["*"]), + visibility = ["//tensorflow:__pkg__"], +) diff --git a/tensorflow/contrib/tensorboard/db/schema.cc b/tensorflow/contrib/tensorboard/db/schema.cc new file mode 100644 index 0000000000000000000000000000000000000000..98fff9e0ae45279f5734ed2eaac8bf46e8ae4b22 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/schema.cc @@ -0,0 +1,409 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/schema.h" + +namespace tensorflow { +namespace { + +class SqliteSchema { + public: + explicit SqliteSchema(std::shared_ptr db) : db_(std::move(db)) {} + + /// \brief Creates Tensors table. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// tag_id: ID of associated Tag. + /// computed_time: Float UNIX timestamp with microsecond precision. + /// In the old summaries system that uses FileWriter, this is the + /// wall time around when tf.Session.run finished. In the new + /// summaries system, it is the wall time of when the tensor was + /// computed. On systems with monotonic clocks, it is calculated + /// by adding the monotonic run duration to Run.started_time. + /// This field is not indexed because, in practice, it should be + /// ordered the same or nearly the same as TensorIndex, so local + /// insertion sort might be more suitable. + /// step: User-supplied number, ordering this tensor in Tag. + /// If NULL then the Tag must have only one Tensor. + /// tensor: Can be an INTEGER (DT_INT64), FLOAT (DT_DOUBLE), or + /// BLOB. The structure of a BLOB is currently undefined, but in + /// essence it is a Snappy tf.TensorProto that spills over into + /// TensorChunks. + Status CreateTensorsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Tensors ( + rowid INTEGER PRIMARY KEY, + tag_id INTEGER NOT NULL, + computed_time REAL, + step INTEGER, + tensor BLOB + ) + )sql"); + } + + /// \brief Creates TensorChunks table. + /// + /// This table can be used to split up a tensor across many rows, + /// which has the advantage of not slowing down table scans on the + /// main table, allowing asynchronous fetching, minimizing copying, + /// and preventing large buffers from being allocated. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// tag_id: ID of associated Tag. + /// step: Same as corresponding Tensors.step. + /// sequence: 1-indexed sequence number for ordering chunks. Please + /// note that the 0th index is Tensors.tensor. + /// chunk: Bytes of next chunk in tensor. + Status CreateTensorChunksTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS TensorChunks ( + rowid INTEGER PRIMARY KEY, + tag_id INTEGER NOT NULL, + step INTEGER, + sequence INTEGER, + chunk BLOB + ) + )sql"); + } + + /// \brief Creates Tags table. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// tag_id: Permanent >0 unique ID. + /// run_id: Optional ID of associated Run. + /// tag_name: The tag field in summary.proto, unique across Run. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the wall time of when the row was inserted into the + /// DB. It may be used as a hint for an archival job. + /// metadata: Optional BLOB of SummaryMetadata proto. + /// display_name: Optional for GUI and defaults to tag_name. + /// summary_description: Optional markdown information. + Status CreateTagsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Tags ( + rowid INTEGER PRIMARY KEY, + run_id INTEGER, + tag_id INTEGER NOT NULL, + tag_name TEXT, + inserted_time DOUBLE, + metadata BLOB, + display_name TEXT, + description TEXT + ) + )sql"); + } + + /// \brief Creates Runs table. + /// + /// This table stores information about runs. Each row usually + /// represents a single attempt at training or testing a TensorFlow + /// model, with a given set of hyper-parameters, whose summaries are + /// written out to a single event logs directory with a monotonic step + /// counter. + /// + /// When a run is deleted from this table, TensorBoard should treat all + /// information associated with it as deleted, even if those rows in + /// different tables still exist. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// run_id: Permanent >0 unique ID. + /// experiment_id: Optional ID of associated Experiment. + /// run_name: User-supplied string, unique across Experiment. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the time the row was inserted into the database. It + /// does not change. + /// started_time: Float UNIX timestamp with µs precision. In the + /// old summaries system that uses FileWriter, this is + /// approximated as the first tf.Event.wall_time. In the new + /// summaries system, it is the wall time of when summary writing + /// started, from the perspective of whichever machine talks to + /// the database. This field will be mutated if the run is + /// restarted. + /// description: Optional markdown information. + /// graph: Snappy tf.GraphDef proto with node field cleared. That + /// field can be recreated using GraphNodes and NodeDefs. + Status CreateRunsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Runs ( + rowid INTEGER PRIMARY KEY, + experiment_id INTEGER, + run_id INTEGER NOT NULL, + run_name TEXT, + inserted_time REAL, + started_time REAL, + description TEXT, + graph BLOB + ) + )sql"); + } + + /// \brief Creates Experiments table. + /// + /// This table stores information about experiments, which are sets of + /// runs. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// user_id: Optional ID of associated User. + /// experiment_id: Permanent >0 unique ID. + /// experiment_name: User-supplied string, unique across User. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the time the row was inserted into the database. It + /// does not change. + /// started_time: Float UNIX timestamp with µs precision. This is + /// the MIN(experiment.started_time, run.started_time) of each + /// Run added to the database. + /// description: Optional markdown information. + Status CreateExperimentsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Experiments ( + rowid INTEGER PRIMARY KEY, + user_id INTEGER, + experiment_id INTEGER NOT NULL, + experiment_name TEXT, + inserted_time REAL, + started_time REAL, + description TEXT + ) + )sql"); + } + + /// \brief Creates Users table. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// user_id: Permanent >0 unique ID. + /// user_name: Unique user name. + /// email: Optional unique email address. + /// inserted_time: Float UNIX timestamp with µs precision. This is + /// always the time the row was inserted into the database. It + /// does not change. + Status CreateUsersTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS Users ( + rowid INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL, + user_name TEXT, + email TEXT, + inserted_time REAL + ) + )sql"); + } + + /// \brief Creates NodeDefs table. + /// + /// This table stores NodeDef protos which define the GraphDef for a + /// Run. This functions like a hash table so rows can be shared by + /// multiple Runs in an Experiment. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// experiment_id: Optional int64 for grouping rows. + /// node_def_id: Permanent >0 unique ID. + /// fingerprint: Optional farmhash::Fingerprint64() of uncompressed + /// node_def bytes, coerced to int64. + /// node_def: BLOB containing a Snappy tf.NodeDef proto. + Status CreateNodeDefsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS NodeDefs ( + rowid INTEGER PRIMARY KEY, + experiment_id INTEGER, + node_def_id INTEGER NOT NULL, + fingerprint INTEGER, + node_def TEXT + ) + )sql"); + } + + /// \brief Creates RunNodeDefs table. + /// + /// Table mapping Runs to NodeDefs. This is used to recreate the node + /// field of the GraphDef proto. + /// + /// Fields: + /// rowid: Ephemeral b-tree ID dictating locality. + /// run_id: Mandatory ID of associated Run. + /// node_def_id: Mandatory ID of associated NodeDef. + Status CreateRunNodeDefsTable() { + return Run(R"sql( + CREATE TABLE IF NOT EXISTS RunNodeDefs ( + rowid INTEGER PRIMARY KEY, + run_id INTEGER NOT NULL, + node_def_id INTEGER NOT NULL + ) + )sql"); + } + + /// \brief Uniquely indexes (tag_id, step) on Tensors table. + Status CreateTensorIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TensorIndex + ON Tensors (tag_id, step) + )sql"); + } + + /// \brief Uniquely indexes (tag_id, step, sequence) on TensorChunks table. + Status CreateTensorChunkIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TensorChunkIndex + ON TensorChunks (tag_id, step, sequence) + )sql"); + } + + /// \brief Uniquely indexes tag_id on Tags table. + Status CreateTagIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TagIdIndex + ON Tags (tag_id) + )sql"); + } + + /// \brief Uniquely indexes run_id on Runs table. + Status CreateRunIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS RunIdIndex + ON Runs (run_id) + )sql"); + } + + /// \brief Uniquely indexes experiment_id on Experiments table. + Status CreateExperimentIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS ExperimentIdIndex + ON Experiments (experiment_id) + )sql"); + } + + /// \brief Uniquely indexes user_id on Users table. + Status CreateUserIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS UserIdIndex + ON Users (user_id) + )sql"); + } + + /// \brief Uniquely indexes node_def_id on NodeDefs table. + Status CreateNodeDefIdIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS NodeDefIdIndex + ON NodeDefs (node_def_id) + )sql"); + } + + /// \brief Uniquely indexes (run_id, tag_name) on Tags table. + Status CreateTagNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS TagNameIndex + ON Tags (run_id, tag_name) + WHERE tag_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes (experiment_id, run_name) on Runs table. + Status CreateRunNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS RunNameIndex + ON Runs (experiment_id, run_name) + WHERE run_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes (user_id, experiment_name) on Experiments table. + Status CreateExperimentNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS ExperimentNameIndex + ON Experiments (user_id, experiment_name) + WHERE experiment_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes user_name on Users table. + Status CreateUserNameIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS UserNameIndex + ON Users (user_name) + WHERE user_name IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes email on Users table. + Status CreateUserEmailIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS UserEmailIndex + ON Users (email) + WHERE email IS NOT NULL + )sql"); + } + + /// \brief Indexes (experiment_id, fingerprint) on NodeDefs table. + Status CreateNodeDefFingerprintIndex() { + return Run(R"sql( + CREATE INDEX IF NOT EXISTS NodeDefFingerprintIndex + ON NodeDefs (experiment_id, fingerprint) + WHERE fingerprint IS NOT NULL + )sql"); + } + + /// \brief Uniquely indexes (run_id, node_def_id) on RunNodeDefs table. + Status CreateRunNodeDefIndex() { + return Run(R"sql( + CREATE UNIQUE INDEX IF NOT EXISTS RunNodeDefIndex + ON RunNodeDefs (run_id, node_def_id) + )sql"); + } + + Status Run(const char* sql) { + auto stmt = db_->Prepare(sql); + TF_RETURN_WITH_CONTEXT_IF_ERROR(stmt.StepAndReset(), sql); + return Status::OK(); + } + + private: + std::shared_ptr db_; +}; + +} // namespace + +Status SetupTensorboardSqliteDb(std::shared_ptr db) { + SqliteSchema s(std::move(db)); + TF_RETURN_IF_ERROR(s.CreateTensorsTable()); + TF_RETURN_IF_ERROR(s.CreateTensorChunksTable()); + TF_RETURN_IF_ERROR(s.CreateTagsTable()); + TF_RETURN_IF_ERROR(s.CreateRunsTable()); + TF_RETURN_IF_ERROR(s.CreateExperimentsTable()); + TF_RETURN_IF_ERROR(s.CreateUsersTable()); + TF_RETURN_IF_ERROR(s.CreateNodeDefsTable()); + TF_RETURN_IF_ERROR(s.CreateRunNodeDefsTable()); + TF_RETURN_IF_ERROR(s.CreateTensorIndex()); + TF_RETURN_IF_ERROR(s.CreateTensorChunkIndex()); + TF_RETURN_IF_ERROR(s.CreateTagIdIndex()); + TF_RETURN_IF_ERROR(s.CreateRunIdIndex()); + TF_RETURN_IF_ERROR(s.CreateExperimentIdIndex()); + TF_RETURN_IF_ERROR(s.CreateUserIdIndex()); + TF_RETURN_IF_ERROR(s.CreateNodeDefIdIndex()); + TF_RETURN_IF_ERROR(s.CreateTagNameIndex()); + TF_RETURN_IF_ERROR(s.CreateRunNameIndex()); + TF_RETURN_IF_ERROR(s.CreateExperimentNameIndex()); + TF_RETURN_IF_ERROR(s.CreateUserNameIndex()); + TF_RETURN_IF_ERROR(s.CreateUserEmailIndex()); + TF_RETURN_IF_ERROR(s.CreateNodeDefFingerprintIndex()); + TF_RETURN_IF_ERROR(s.CreateRunNodeDefIndex()); + return Status::OK(); +} + +} // namespace tensorflow diff --git a/tensorflow/contrib/tensorboard/db/schema.h b/tensorflow/contrib/tensorboard/db/schema.h new file mode 100644 index 0000000000000000000000000000000000000000..900c10298ce0a69b92f7528db9742517243c3c51 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/schema.h @@ -0,0 +1,33 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ +#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ + +#include + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/db/sqlite.h" + +namespace tensorflow { + +/// \brief Creates TensorBoard SQLite tables and indexes. +/// +/// If they are already created, this has no effect. If schema +/// migrations are necessary, they will be performed with logging. +Status SetupTensorboardSqliteDb(std::shared_ptr db); + +} // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SCHEMA_H_ diff --git a/tensorflow/contrib/tensorboard/db/schema_test.cc b/tensorflow/contrib/tensorboard/db/schema_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..463c4e59e7e76e6460b7ddfbd92262ac249aa9ed --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/schema_test.cc @@ -0,0 +1,31 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/schema.h" + +#include + +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +TEST(SchemaTest, SmokeTestTensorboardSchema) { + auto db = Sqlite::Open(":memory:").ValueOrDie(); + TF_ASSERT_OK(SetupTensorboardSqliteDb(db)); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.cc b/tensorflow/contrib/tensorboard/db/summary_db_writer.cc new file mode 100644 index 0000000000000000000000000000000000000000..df64e36305529a67f9573e9d26cc0dfc506d324f --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/summary_db_writer.cc @@ -0,0 +1,279 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h" + +#include "tensorflow/contrib/tensorboard/db/schema.h" +#include "tensorflow/core/lib/db/sqlite.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/lib/strings/stringprintf.h" +#include "tensorflow/core/platform/snappy.h" + +namespace tensorflow { +namespace { + +int64 MakeRandomId() { + int64 id = static_cast(random::New64() & ((1ULL << 63) - 1)); + if (id == 0) { + ++id; + } + return id; +} + +class SummaryDbWriter : public SummaryWriterInterface { + public: + SummaryDbWriter(Env* env, std::shared_ptr db) + : SummaryWriterInterface(), env_(env), db_(std::move(db)), run_id_(-1) {} + ~SummaryDbWriter() override {} + + Status Initialize(const string& experiment_name, const string& run_name, + const string& user_name) { + mutex_lock ml(mu_); + insert_tensor_ = db_->Prepare(R"sql( + INSERT OR REPLACE INTO Tensors (tag_id, step, computed_time, tensor) + VALUES (?, ?, ?, ?) + )sql"); + update_metadata_ = db_->Prepare(R"sql( + UPDATE Tags SET metadata = ? WHERE tag_id = ? + )sql"); + experiment_name_ = experiment_name; + run_name_ = run_name; + user_name_ = user_name; + return Status::OK(); + } + + // TODO(@jart): Use transactions that COMMIT on Flush() + // TODO(@jart): Retry Commit() on SQLITE_BUSY with exponential back-off. + Status Flush() override { return Status::OK(); } + + Status WriteTensor(int64 global_step, Tensor t, const string& tag, + const string& serialized_metadata) override { + mutex_lock ml(mu_); + TF_RETURN_IF_ERROR(InitializeParents()); + // TODO(@jart): Memoize tag_id. + int64 tag_id; + TF_RETURN_IF_ERROR(GetTagId(run_id_, tag, &tag_id)); + if (!serialized_metadata.empty()) { + // TODO(@jart): Only update metadata for first tensor. + update_metadata_.BindBlobUnsafe(1, serialized_metadata); + update_metadata_.BindInt(2, tag_id); + TF_RETURN_IF_ERROR(update_metadata_.StepAndReset()); + } + // TODO(@jart): Lease blocks of rowids and *_ids to minimize fragmentation. + // TODO(@jart): Check for random ID collisions without needing txn retry. + insert_tensor_.BindInt(1, tag_id); + insert_tensor_.BindInt(2, global_step); + insert_tensor_.BindDouble(3, GetWallTime()); + switch (t.dtype()) { + case DT_INT64: + insert_tensor_.BindInt(4, t.scalar()()); + break; + case DT_DOUBLE: + insert_tensor_.BindDouble(4, t.scalar()()); + break; + default: + TF_RETURN_IF_ERROR(BindTensor(t)); + break; + } + TF_RETURN_IF_ERROR(insert_tensor_.StepAndReset()); + return Status::OK(); + } + + Status WriteEvent(std::unique_ptr e) override { + // TODO(@jart): This will be used to load event logs. + return errors::Unimplemented("WriteEvent"); + } + + Status WriteScalar(int64 global_step, Tensor t, const string& tag) override { + // TODO(@jart): Unlike WriteTensor, this method would be granted leniency + // to change the dtype if it saves storage space. For example, + // DT_UINT32 would be stored in the database as an INTEGER + // rather than a serialized BLOB. But when reading it back, + // the dtype would become DT_INT64. + return errors::Unimplemented("WriteScalar"); + } + + Status WriteHistogram(int64 global_step, Tensor t, + const string& tag) override { + return errors::Unimplemented( + "SummaryDbWriter::WriteHistogram not supported. Please use ", + "tensorboard.summary.histogram() instead."); + } + + Status WriteImage(int64 global_step, Tensor tensor, const string& tag, + int max_images, Tensor bad_color) override { + return errors::Unimplemented( + "SummaryDbWriter::WriteImage not supported. Please use ", + "tensorboard.summary.image() instead."); + } + + Status WriteAudio(int64 global_step, Tensor tensor, const string& tag, + int max_outputs, float sample_rate) override { + return errors::Unimplemented( + "SummaryDbWriter::WriteAudio not supported. Please use ", + "tensorboard.summary.audio() instead."); + } + + string DebugString() override { return "SummaryDbWriter"; } + + private: + double GetWallTime() { + // TODO(@jart): Follow precise definitions for time laid out in schema. + // TODO(@jart): Use monotonic clock from gRPC codebase. + return static_cast(env_->NowMicros()) / 1.0e6; + } + + Status BindTensor(const Tensor& t) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // TODO(@jart): Make portable between little and big endian systems. + // TODO(@jart): Use TensorChunks with minimal copying for big tensors. + TensorProto p; + t.AsProtoTensorContent(&p); + string encoded; + if (!p.SerializeToString(&encoded)) { + return errors::DataLoss("SerializeToString failed"); + } + // TODO(@jart): Put byte at beginning of blob to indicate encoding. + // TODO(@jart): Allow crunch tool to re-compress with zlib instead. + string compressed; + if (!port::Snappy_Compress(encoded.data(), encoded.size(), &compressed)) { + return errors::FailedPrecondition("TensorBase needs Snappy"); + } + insert_tensor_.BindBlobUnsafe(4, compressed); + return Status::OK(); + } + + Status InitializeParents() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (run_id_ >= 0) { + return Status::OK(); + } + int64 user_id; + TF_RETURN_IF_ERROR(GetUserId(user_name_, &user_id)); + int64 experiment_id; + TF_RETURN_IF_ERROR( + GetExperimentId(user_id, experiment_name_, &experiment_id)); + TF_RETURN_IF_ERROR(GetRunId(experiment_id, run_name_, &run_id_)); + return Status::OK(); + } + + Status GetUserId(const string& user_name, int64* user_id) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (user_name.empty()) { + *user_id = 0LL; + return Status::OK(); + } + SqliteStatement get_user_id = db_->Prepare(R"sql( + SELECT user_id FROM Users WHERE user_name = ? + )sql"); + get_user_id.BindText(1, user_name); + bool is_done; + TF_RETURN_IF_ERROR(get_user_id.Step(&is_done)); + if (!is_done) { + *user_id = get_user_id.ColumnInt(0); + } else { + *user_id = MakeRandomId(); + SqliteStatement insert_user = db_->Prepare(R"sql( + INSERT INTO Users (user_id, user_name, inserted_time) VALUES (?, ?, ?) + )sql"); + insert_user.BindInt(1, *user_id); + insert_user.BindText(2, user_name); + insert_user.BindDouble(3, GetWallTime()); + TF_RETURN_IF_ERROR(insert_user.StepAndReset()); + } + return Status::OK(); + } + + Status GetExperimentId(int64 user_id, const string& experiment_name, + int64* experiment_id) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // TODO(@jart): Compute started_time. + return GetId("Experiments", "user_id", user_id, "experiment_name", + experiment_name, "experiment_id", experiment_id); + } + + Status GetRunId(int64 experiment_id, const string& run_name, int64* run_id) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + // TODO(@jart): Compute started_time. + return GetId("Runs", "experiment_id", experiment_id, "run_name", run_name, + "run_id", run_id); + } + + Status GetTagId(int64 run_id, const string& tag_name, int64* tag_id) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return GetId("Tags", "run_id", run_id, "tag_name", tag_name, "tag_id", + tag_id); + } + + Status GetId(const char* table, const char* parent_id_field, int64 parent_id, + const char* name_field, const string& name, const char* id_field, + int64* id) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (name.empty()) { + *id = 0LL; + return Status::OK(); + } + SqliteStatement select = db_->Prepare( + strings::Printf("SELECT %s FROM %s WHERE %s = ? AND %s = ?", id_field, + table, parent_id_field, name_field)); + if (parent_id > 0) { + select.BindInt(1, parent_id); + } + select.BindText(2, name); + bool is_done; + TF_RETURN_IF_ERROR(select.Step(&is_done)); + if (!is_done) { + *id = select.ColumnInt(0); + } else { + *id = MakeRandomId(); + SqliteStatement insert = db_->Prepare(strings::Printf( + "INSERT INTO %s (%s, %s, %s, inserted_time) VALUES (?, ?, ?, ?)", + table, parent_id_field, id_field, name_field)); + if (parent_id > 0) { + insert.BindInt(1, parent_id); + } + insert.BindInt(2, *id); + insert.BindText(3, name); + insert.BindDouble(4, GetWallTime()); + TF_RETURN_IF_ERROR(insert.StepAndReset()); + } + return Status::OK(); + } + + mutex mu_; + Env* env_; + std::shared_ptr db_ GUARDED_BY(mu_); + SqliteStatement insert_tensor_ GUARDED_BY(mu_); + SqliteStatement update_metadata_ GUARDED_BY(mu_); + string user_name_ GUARDED_BY(mu_); + string experiment_name_ GUARDED_BY(mu_); + string run_name_ GUARDED_BY(mu_); + int64 run_id_ GUARDED_BY(mu_); +}; + +} // namespace + +Status CreateSummaryDbWriter(std::shared_ptr db, + const string& experiment_name, + const string& run_name, const string& user_name, + Env* env, SummaryWriterInterface** result) { + TF_RETURN_IF_ERROR(SetupTensorboardSqliteDb(db)); + SummaryDbWriter* w = new SummaryDbWriter(env, std::move(db)); + const Status s = w->Initialize(experiment_name, run_name, user_name); + if (!s.ok()) { + w->Unref(); + *result = nullptr; + return s; + } + *result = w; + return Status::OK(); +} + +} // namespace tensorflow diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer.h b/tensorflow/contrib/tensorboard/db/summary_db_writer.h new file mode 100644 index 0000000000000000000000000000000000000000..74f61e50b7cdf4b4151162a2e1e5e0af0d468be2 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/summary_db_writer.h @@ -0,0 +1,42 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_ +#define TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_ + +#include "tensorflow/core/kernels/summary_interface.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/db/sqlite.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +/// \brief Creates SQLite SummaryWriterInterface. +/// +/// This can be used to write tensors from the execution graph directly +/// to a database. The schema will be created automatically, but only +/// if necessary. Entries in the Users, Experiments, and Runs tables +/// will be created automatically if they don't already exist. +/// +/// Please note that the type signature of this function may change in +/// the future if support for other DBs is added to core. +Status CreateSummaryDbWriter(std::shared_ptr db, + const string& experiment_name, + const string& run_name, const string& user_name, + Env* env, SummaryWriterInterface** result); + +} // namespace tensorflow + +#endif // TENSORFLOW_CONTRIB_TENSORBOARD_DB_SUMMARY_DB_WRITER_H_ diff --git a/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc b/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..d32904f97c4172ded51a00dc076630b598494716 --- /dev/null +++ b/tensorflow/contrib/tensorboard/db/summary_db_writer_test.cc @@ -0,0 +1,162 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/contrib/tensorboard/db/summary_db_writer.h" + +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/db/sqlite.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +Tensor MakeScalarInt64(int64 x) { + Tensor t(DT_INT64, TensorShape({})); + t.scalar()() = x; + return t; +} + +class FakeClockEnv : public EnvWrapper { + public: + FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {} + void AdvanceByMillis(const uint64 millis) { current_millis_ += millis; } + uint64 NowMicros() override { return current_millis_ * 1000; } + uint64 NowSeconds() override { return current_millis_ * 1000; } + + private: + uint64 current_millis_; +}; + +class SummaryDbWriterTest : public ::testing::Test { + protected: + void SetUp() override { db_ = Sqlite::Open("file::memory:").ValueOrDie(); } + + void TearDown() override { + if (writer_ != nullptr) { + writer_->Unref(); + writer_ = nullptr; + } + } + + int64 QueryInt(const string& sql) { + SqliteStatement stmt = db_->Prepare(sql); + bool is_done; + Status s = stmt.Step(&is_done); + if (!s.ok() || is_done) { + LOG(ERROR) << s << " due to " << sql; + return -1; + } + return stmt.ColumnInt(0); + } + + double QueryDouble(const string& sql) { + SqliteStatement stmt = db_->Prepare(sql); + bool is_done; + Status s = stmt.Step(&is_done); + if (!s.ok() || is_done) { + LOG(ERROR) << s << " due to " << sql; + return -1; + } + return stmt.ColumnDouble(0); + } + + string QueryString(const string& sql) { + SqliteStatement stmt = db_->Prepare(sql); + bool is_done; + Status s = stmt.Step(&is_done); + if (!s.ok() || is_done) { + LOG(ERROR) << s << " due to " << sql; + return "MISSINGNO"; + } + return stmt.ColumnString(0); + } + + FakeClockEnv env_; + std::shared_ptr db_; + SummaryWriterInterface* writer_ = nullptr; +}; + +TEST_F(SummaryDbWriterTest, NothingWritten_NoRowsCreated) { + TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_, + &writer_)); + TF_ASSERT_OK(writer_->Flush()); + writer_->Unref(); + writer_ = nullptr; + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Users")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Experiments")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Runs")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Tags")); + EXPECT_EQ(0LL, QueryInt("SELECT COUNT(*) FROM Tensors")); +} + +TEST_F(SummaryDbWriterTest, TensorsWritten_RowsGetInitialized) { + TF_ASSERT_OK(CreateSummaryDbWriter(db_, "mad-science", "train", "jart", &env_, + &writer_)); + env_.AdvanceByMillis(23); + TF_ASSERT_OK(writer_->WriteTensor(1, MakeScalarInt64(123LL), "taggy", + "this-is-metaaa")); + env_.AdvanceByMillis(23); + TF_ASSERT_OK(writer_->WriteTensor(2, MakeScalarInt64(314LL), "taggy", "")); + TF_ASSERT_OK(writer_->Flush()); + + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Users")); + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Experiments")); + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Runs")); + ASSERT_EQ(1LL, QueryInt("SELECT COUNT(*) FROM Tags")); + ASSERT_EQ(2LL, QueryInt("SELECT COUNT(*) FROM Tensors")); + + int64 user_id = QueryInt("SELECT user_id FROM Users"); + int64 experiment_id = QueryInt("SELECT experiment_id FROM Experiments"); + int64 run_id = QueryInt("SELECT run_id FROM Runs"); + int64 tag_id = QueryInt("SELECT tag_id FROM Tags"); + EXPECT_LT(0LL, user_id); + EXPECT_LT(0LL, experiment_id); + EXPECT_LT(0LL, run_id); + EXPECT_LT(0LL, tag_id); + + EXPECT_EQ("jart", QueryString("SELECT user_name FROM Users")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Users")); + + EXPECT_EQ(user_id, QueryInt("SELECT user_id FROM Experiments")); + EXPECT_EQ("mad-science", + QueryString("SELECT experiment_name FROM Experiments")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Experiments")); + + EXPECT_EQ(experiment_id, QueryInt("SELECT experiment_id FROM Runs")); + EXPECT_EQ("train", QueryString("SELECT run_name FROM Runs")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Runs")); + + EXPECT_EQ(run_id, QueryInt("SELECT run_id FROM Tags")); + EXPECT_EQ("taggy", QueryString("SELECT tag_name FROM Tags")); + EXPECT_EQ(0.023, QueryDouble("SELECT inserted_time FROM Tags")); + EXPECT_EQ("this-is-metaaa", QueryString("SELECT metadata FROM Tags")); + + EXPECT_EQ(tag_id, QueryInt("SELECT tag_id FROM Tensors WHERE step = 1")); + EXPECT_EQ(0.023, + QueryDouble("SELECT computed_time FROM Tensors WHERE step = 1")); + EXPECT_EQ("this-is-metaaa", QueryString("SELECT metadata FROM Tags")); + EXPECT_FALSE( + QueryString("SELECT tensor FROM Tensors WHERE step = 1").empty()); + + EXPECT_EQ(tag_id, QueryInt("SELECT tag_id FROM Tensors WHERE step = 2")); + EXPECT_EQ(0.046, + QueryDouble("SELECT computed_time FROM Tensors WHERE step = 2")); + EXPECT_EQ("this-is-metaaa", QueryString("SELECT metadata FROM Tags")); + EXPECT_FALSE( + QueryString("SELECT tensor FROM Tensors WHERE step = 2").empty()); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/contrib/text/BUILD b/tensorflow/contrib/text/BUILD index 8a2cb28684fe5151176b00fbcfaa64626ec18c38..698fdd830f57eb64c3c4119371f545908bf726e5 100644 --- a/tensorflow/contrib/text/BUILD +++ b/tensorflow/contrib/text/BUILD @@ -36,15 +36,21 @@ tf_custom_op_py_library( srcs_version = "PY2AND3", deps = [ ":gen_skip_gram_ops", + "//tensorflow/contrib/lookup:lookup_py", "//tensorflow/contrib/util:util_py", "//tensorflow/python:array_ops", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", + "//tensorflow/python:dtypes", "//tensorflow/python:framework", + "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:ops", "//tensorflow/python:platform", + "//tensorflow/python:random_ops", + "//tensorflow/python:random_seed", "//tensorflow/python:training", + "//tensorflow/python:util", ], ) diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD index 222a77c4898bf705f98f98fba841bbfff5e852cc..755b0657e9fb29c167911407cee340ac7e3e9b7a 100644 --- a/tensorflow/contrib/timeseries/examples/BUILD +++ b/tensorflow/contrib/timeseries/examples/BUILD @@ -88,6 +88,8 @@ py_binary( tags = ["no_pip"], deps = [ "//tensorflow:tensorflow_py", + "//tensorflow/contrib/timeseries/python/timeseries:estimators", + "//tensorflow/contrib/timeseries/python/timeseries:model", "//third_party/py/numpy", ], ) @@ -98,7 +100,10 @@ py_test( srcs = ["lstm_test.py"], srcs_version = "PY2AND3", tags = ["notsan"], - deps = [":lstm"], + deps = [ + ":lstm", + "//tensorflow/python:client_testlib", + ], ) filegroup( diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD index 76e8ccc62a2d34acf333515043d20afc456b1924..5f04eb2f5a4af031ad19662b05a8a2396299925d 100644 --- a/tensorflow/contrib/timeseries/python/timeseries/BUILD +++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD @@ -105,6 +105,7 @@ py_test( tags = [ "no_pip_gpu", # b/63391119 "nomsan", # Takes too long to run. + "notsan", # b/67865658 ], deps = [ ":ar_model", @@ -137,15 +138,13 @@ py_library( "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", - "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", - "//tensorflow/python:nn_ops", "//tensorflow/python:state_ops", "//tensorflow/python:util", "//tensorflow/python:variable_scope", "//tensorflow/python/estimator:estimator_py", "//tensorflow/python/estimator:export", - "//third_party/py/numpy", + "//tensorflow/python/estimator:head", ], ) @@ -183,7 +182,6 @@ py_library( srcs_version = "PY2AND3", deps = [ ":feature_keys", - "//tensorflow/contrib/framework:framework_py", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", "//tensorflow/python:init_ops", @@ -206,7 +204,6 @@ py_test( ":model_utils", "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", - "//tensorflow/python:variables", ], ) @@ -326,11 +323,11 @@ py_library( ":input_pipeline", ":state_management", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:framework_ops", "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:random_seed", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:util", "//tensorflow/python:variables", @@ -379,10 +376,10 @@ py_test( ":input_pipeline", ":test_utils", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_ops", "//tensorflow/python:platform", + "//tensorflow/python:session", "//tensorflow/python:training", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD index 970fc97605057bd65fc5c0796f6a6a5f0a27e458..c89596734c738467c58e845328e396c3f2eb999a 100644 --- a/tensorflow/contrib/tpu/BUILD +++ b/tensorflow/contrib/tpu/BUILD @@ -30,6 +30,18 @@ cc_library( ], ) +py_library( + name = "tpu_test_util", + srcs = [ + "python/tpu/test_util.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":tpu_lib", + ":tpu_py", + ], +) + py_library( name = "tpu_estimator", srcs = [ diff --git a/tensorflow/contrib/tpu/python/tpu/test_util.py b/tensorflow/contrib/tpu/python/tpu/test_util.py new file mode 100644 index 0000000000000000000000000000000000000000..f30c27f1298e2389fe0daefdd4eece5a03a6976c --- /dev/null +++ b/tensorflow/contrib/tpu/python/tpu/test_util.py @@ -0,0 +1,153 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =================================================================== +"""Utilities to ease testing on TPU devices.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.contrib.tpu.python.tpu import tpu + +from tensorflow.python.client import session +from tensorflow.python.framework import errors +from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import gen_array_ops +from tensorflow.python.ops import variables + + +def has_tpu(): + """Check if a TPU device is available. + + Device enumeration via `device_lib` currently fails for TPU systems. + (http://b/68333779). To work around this, we determine the existence of a + TPU by a successful call to `initialize_system`. + + Returns: + boolean, True if a TPU device is available, otherwise False. + """ + def _check(): + with session.Session() as sess: + sess.run(tpu.initialize_system()) + sess.run(tpu.shutdown_system()) + + try: + _check() + return True + except errors.OpError as _: + return False + + +def _available_devices(): + devices = ["cpu"] + if not test_util.gpu_device_name(): + devices.append("gpu") + + if has_tpu(): + devices.append("tpu") + + return tuple(devices) + + +class TPUTestCase(test_util.TensorFlowTestCase): + """Adds helpers for testing on TPU devices to `TensorFlowTestCase`. + + Example usage: + + ``` + def model_fn(features): + return tf.reduce_sum(features * 2) + + class ModelTests(test_util.TPUTestCase): + def test_sum(self): + v = np.random.randn(10, 10).astype("float32") + self.assert_device_output(model_fn, [v], (v*2).sum(), + devices=("cpu", "tpu")) + ``` + """ + + def __init__(self, methodName="runTest"): # pylint: disable=invalid-name + super(TPUTestCase, self).__init__(methodName) + self._available_devices = _available_devices() + + def run_on_device(self, model_fn, model_inputs, device): + """Runs `model_fn` on the given device. + + Raises an exception if no such device is available. `model_fn` should + return one or more tensors as a list or tuple. + + Args: + model_fn: Function returning one or more tensors. + model_inputs: An iterable of Numpy arrays or scalars. + These will be passed as arguments to `model_fn`. + device: Device to run on. One of ("tpu", "gpu", "cpu"). + + Returns: + Output from the model function. + """ + def _make_placeholders(): + return dict( + [(gen_array_ops.placeholder_with_default(v, v.shape), v) + for v in model_inputs]) + + if device == "tpu": + with self.test_session(graph=ops.Graph()) as sess: + placeholders = _make_placeholders() + tpu_computation = tpu.rewrite(model_fn, placeholders.keys()) + sess.run(tpu.initialize_system()) + sess.run(variables.global_variables_initializer()) + result = sess.run(tpu_computation, placeholders) + sess.run(tpu.shutdown_system()) + # TODO(b/36891278): supports non-flat returns lists in tpu.rewrite(). + if len(result) == 1: + return result[0] + return result + elif device == "gpu": + with self.test_session(graph=ops.Graph(), use_gpu=True) as sess: + placeholders = _make_placeholders() + sess.run(variables.global_variables_initializer()) + return sess.run(model_fn(placeholders.keys()), placeholders) + elif device == "cpu": + # TODO(power) -- will this interact poorly with cached GPU sessions? + with self.test_session(graph=ops.Graph(), use_gpu=False) as sess: + placeholders = _make_placeholders() + sess.run(variables.global_variables_initializer()) + return sess.run(model_fn(placeholders.keys()), placeholders) + + def _compare_values(self, actual_outputs, expected_outputs): + if isinstance(expected_outputs, (list, tuple)): + for a, b in zip(actual_outputs, expected_outputs): + self.assertAllCloseAccordingToType(a, b) + else: + self.assertAllCloseAccordingToType(actual_outputs, expected_outputs) + + def assert_device_output(self, model_fn, model_inputs, expected_outputs, + devices=("cpu", "gpu", "tpu")): + """Run `model_fn` on the given devices. + + Results are compared via `assertAllCloseAccordingToType`. + + Args: + model_fn: Function returning one or more tensors + model_inputs: Numpy arrays or scalars passed as arguments to model_fn + expected_outputs: Numpy arrays or scalars to compare against. + devices: Set of devices to run on. If a device is not available, tests + will be skipped for that device. + """ + devices = set(devices).intersection(self._available_devices) + + for device in devices: + device_out = self.run_on_device(model_fn, model_inputs, device=device) + self._compare_values(device_out, expected_outputs) diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py index fa5760953dfa9353a04f9af49b320f57a73cc275..338a4304f3272f3486c88e6e2aeb90fec15e4f58 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu.py @@ -146,6 +146,14 @@ class TPUReplicateContext(control_flow_ops.ControlFlowContext): if self._outer_context: self._outer_context.AddInnerOp(op) + @property + def grad_state(self): + # Define the gradient loop state associated with the TPUReplicateContext to + # be None as the TPUReplicateContext does not get nested nor does the + # grad_state outside the TPUReplicateContext affect the graph inside so the + # grad_state should be as if this is the top-level gradient state. + return None + def replicate(computation, inputs=None, diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py index 0a3be8503a81149c4540ffb5819ae55b30216f97..3965c087a18dc18298703fad9b1dda9c85c56271 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py @@ -27,7 +27,10 @@ from tensorflow.python.estimator import run_config as run_config_lib class TPUConfig( collections.namedtuple('TPUConfig', [ - 'iterations_per_loop', 'num_shards', 'per_host_input_for_training' + 'iterations_per_loop', + 'num_shards', + 'per_host_input_for_training', + 'tpu_job_name', ])): """TPU related configuration required by `TPUEstimator`. @@ -46,12 +49,17 @@ class TPUConfig( that this only works for single-host TPU training now (tracked in b/67051042). For multi-host, please use Per-Core, i.e., `False` for `per_host_input_for_training`. + tpu_job_name: The name of the TPU job. Typically, this name is auto-inferred + within TPUEstimator, however when using ClusterSpec propagation in more + esoteric cluster configurations, you may need to specify the job name as a + string. """ def __new__(cls, iterations_per_loop=2, num_shards=2, - per_host_input_for_training=True): + per_host_input_for_training=True, + tpu_job_name=None): # Check iterations_per_loop. util_lib.check_positive_integer(iterations_per_loop, @@ -59,33 +67,35 @@ class TPUConfig( # Check num_shards. util_lib.check_positive_integer(num_shards, 'TPUConfig num_shards') - return super(TPUConfig, cls).__new__( cls, iterations_per_loop=iterations_per_loop, num_shards=num_shards, - per_host_input_for_training=per_host_input_for_training) + per_host_input_for_training=per_host_input_for_training, + tpu_job_name=tpu_job_name) class RunConfig(run_config_lib.RunConfig): """RunConfig with TPU support.""" - def __init__(self, tpu_config=None, evaluation_master='', master='', - tf_random_seed=None, **kwargs): + def __init__(self, tpu_config=None, evaluation_master=None, master='', + **kwargs): """Constructs a RunConfig. Args: tpu_config: the TPUConfig that specifies TPU-specific configuration. evaluation_master: a string. The address of the master to use for eval. + Defaults to master if not set. master: a string. The address of the master to use for training. tf_random_seed: an int. Sets the TensorFlow random seed. Defaults to None, which initializes it randomly based on the environment. """ - # We change the default random seed to None because that's a better default. - kwargs['tf_random_seed'] = tf_random_seed super(RunConfig, self).__init__(**kwargs) self._tpu_config = tpu_config or TPUConfig() - self._evaluation_master = evaluation_master + if evaluation_master is None: + self._evaluation_master = master + else: + self._evaluation_master = evaluation_master self._master = master @property diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py index 43f9defd54c1f933895c73d461c326a0c54f7de5..5a3b8314291951b5dfce091dccb0dc9e5f7af3b5 100644 --- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py +++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py @@ -20,6 +20,7 @@ from __future__ import division from __future__ import print_function import collections +from contextlib import contextmanager import copy import threading import six @@ -38,6 +39,7 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.python.estimator import estimator as estimator_lib from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator import util +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -57,12 +59,15 @@ from tensorflow.python.training import training_util _INITIAL_LOSS = 1e7 _ZERO_LOSS = 0. -_DEFAULT_NAME_SCOPE = 'tpu_estimator' +_TPU_ESTIMATOR = 'tpu_estimator' _ITERATIONS_PER_LOOP_VAR = 'iterations_per_loop' _BATCH_SIZE_KEY = 'batch_size' _CROSS_REPLICA_SUM_OP = 'CrossReplicaSum' _RESERVED_PARAMS_KEYS = [_BATCH_SIZE_KEY] +# TODO(b/65703635): Flip the value and remove all dead code. +_WRAP_INPUT_FN_INTO_WHILE_LOOP = False + def _create_global_step(graph): graph = graph or ops.get_default_graph() @@ -81,17 +86,25 @@ def _create_global_step(graph): ops.GraphKeys.GLOBAL_STEP]) -def _create_iterations_per_loop(): - with variable_scope.variable_scope(_DEFAULT_NAME_SCOPE, - reuse=variable_scope.AUTO_REUSE): - return variable_scope.get_variable( - _ITERATIONS_PER_LOOP_VAR, - initializer=init_ops.zeros_initializer(), - shape=[], - dtype=dtypes.int32, - trainable=False, - collections=[], - use_resource=True) +def _create_or_get_iterations_per_loop(): + graph = ops.get_default_graph() + iter_vars = graph.get_collection(_TPU_ESTIMATOR) + if len(iter_vars) == 1: + return iter_vars[0] + elif len(iter_vars) > 1: + raise RuntimeError('Multiple iterations_per_loop_var in collection.') + + with ops.colocate_with(training_util.get_global_step()): + with variable_scope.variable_scope(_TPU_ESTIMATOR, + reuse=variable_scope.AUTO_REUSE): + return variable_scope.get_variable( + _ITERATIONS_PER_LOOP_VAR, + initializer=init_ops.zeros_initializer(), + shape=[], + dtype=dtypes.int32, + trainable=False, + collections=[_TPU_ESTIMATOR], + use_resource=True) def _sync_variables_ops(): @@ -122,26 +135,214 @@ def _increase_eval_step_op(iterations_per_loop): use_locking=True) -def _tpu_job(run_config, mode): - # The tpu job is determined by the run_config. Right now, this method is - # required as tpu_config is not part of the RunConfig. - master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL - else run_config.master) - return None if master in ['', 'local'] else 'tpu_worker' +_DEFAULT_JOB_NAME = 'tpu_worker' +_DEFAULT_COORDINATOR_JOB_NAME = 'coordinator' +_LOCAL_MASTERS = ('', 'local') + + +class _TPUContext(object): + """A context holds immutable states of TPU computation. + + This immutable object holds TPUEstimator config, train/eval batch size, and + `TPUEstimator.use_tpu`, which is expected to be passed around. It also + provides utility functions, basded on the current state, to determine other + information commonly required by TPU computation, such as TPU device names, + TPU hosts, shard batch size, etc. + + N.B. As `mode` is not immutable state in Estimator, but essential to + distinguish between TPU training and evaluation, a common usage for + _TPUContext with `mode` is as follows: + ``` + with _ctx.with_mode(mode) as ctx: + if ctx.is_running_on_cpu(): + ... + ``` + """ + + def __init__(self, config, train_batch_size, eval_batch_size, use_tpu): + self._config = config + self._train_batch_size = train_batch_size + self._eval_batch_size = eval_batch_size + self._use_tpu = use_tpu + self._num_shards_or_none = self._config.tpu_config.num_shards + self._mode = None + + def _assert_mode(self): + if self._mode is None: + raise RuntimeError( + '`mode` needs to be set via contextmanager `with_mode`.') + return self._mode + + @property + def num_of_cores_per_host(self): + num_cores = self.num_cores + return min(num_cores, 8) + + @contextmanager + def with_mode(self, mode): + new_ctx = copy.copy(self) # Shallow copy is enough. + new_ctx._mode = mode # pylint: disable=protected-access + yield new_ctx + + @property + def mode(self): + return self._assert_mode() + + @property + def num_cores(self): + # TODO(xiejw): Adds lazy num_shards initialization. + return self._num_shards_or_none + + @property + def num_hosts(self): + return self.num_cores // self.num_of_cores_per_host + + @property + def config(self): + return self._config + + def is_input_sharded_per_core(self): + """Return true if input_fn is invoked per-core (other than per-host).""" + self._assert_mode() + return (self._mode == model_fn_lib.ModeKeys.TRAIN and + not self._config.tpu_config.per_host_input_for_training) + def is_running_on_cpu(self): + """Determines whether the input_fn and model_fn should be invoked on CPU.""" + mode = self._assert_mode() + return ((not self._use_tpu) or mode == model_fn_lib.ModeKeys.PREDICT or + (mode == model_fn_lib.ModeKeys.EVAL and + self._eval_batch_size is None)) -def _is_running_on_cpu(use_tpu, mode, eval_batch_size): - """Determines whether the input_fn and model_fn should be invoked on CPU.""" - return ((not use_tpu) or mode == model_fn_lib.ModeKeys.PREDICT or - (mode == model_fn_lib.ModeKeys.EVAL and eval_batch_size is None)) + @property + def batch_size_for_input_fn(self): + """Returns the shard batch size for `input_fn`.""" + mode = self._assert_mode() + # Special case for eval. + if mode == model_fn_lib.ModeKeys.EVAL and self._eval_batch_size is None: + return None + if self.is_running_on_cpu(): + if mode == model_fn_lib.ModeKeys.TRAIN: + return self._train_batch_size + if mode == model_fn_lib.ModeKeys.EVAL: + return self._eval_batch_size + return None + global_batch_size = (self._train_batch_size if + mode == model_fn_lib.ModeKeys.TRAIN + else self._eval_batch_size) + # On TPU + return (global_batch_size // self.num_cores + if self.is_input_sharded_per_core() else global_batch_size) -def _per_shard_batch_size(global_batch_size, run_config, use_tpu): - """Returns the batch size for each shard.""" - if use_tpu: - return global_batch_size // run_config.tpu_config.num_shards - else: - return global_batch_size + @property + def batch_size_for_model_fn(self): + """Returns the shard batch size for `model_fn`.""" + mode = self._assert_mode() + # Special case for eval. + if mode == model_fn_lib.ModeKeys.EVAL and self._eval_batch_size is None: + return None + if self.is_running_on_cpu(): + if mode == model_fn_lib.ModeKeys.TRAIN: + return self._train_batch_size + if mode == model_fn_lib.ModeKeys.EVAL: + return self._eval_batch_size + return None + + # On TPU. always sharded per core. + if mode == model_fn_lib.ModeKeys.TRAIN: + return self._train_batch_size // self.num_cores + else: + return self._eval_batch_size // self.num_cores + + @property + def master_job(self): + """Returns the job name to use to place TPU computations on. + + Returns: + A string containing the job name, or None if no job should be specified. + + Raises: + ValueError: If the user needs to specify a tpu_job_name, because we are + unable to infer the job name automatically, or if the user-specified job + names are inappropriate. + """ + run_config = self._config + # If the user specifies the tpu_job_name, use that. + if run_config.tpu_config.tpu_job_name: + return run_config.tpu_config.tpu_job_name + + # The tpu job is determined by the run_config. Right now, this method is + # required as tpu_config is not part of the RunConfig. + mode = self._assert_mode() + master = (run_config.evaluation_master if mode == model_fn_lib.ModeKeys.EVAL + else run_config.master) + if master in _LOCAL_MASTERS: + return None + + if (not run_config.session_config or + not run_config.session_config.cluster_def.job): + return _DEFAULT_JOB_NAME + cluster_def = run_config.session_config.cluster_def + job_names = set([job.name for job in cluster_def.job]) + if _DEFAULT_JOB_NAME in job_names: + # b/37868888 tracks allowing ClusterSpec propagation to reuse job names. + raise ValueError('Currently, tpu_worker is not an allowed job name.') + if len(job_names) == 1: + return cluster_def.job[0].name + if len(job_names) == 2: + if _DEFAULT_COORDINATOR_JOB_NAME in job_names: + job_names.remove(_DEFAULT_COORDINATOR_JOB_NAME) + return job_names.pop() + # TODO(b/67716447): Include more sophisticated heuristics. + raise ValueError( + 'Could not infer TPU job name. Please specify a tpu_job_name as part ' + 'of your TPUConfig.') + + @property + def tpu_host_placement_function(self): + """Returns the TPU host place function.""" + master = self.master_job + def _placement_function(_sentinal=None, core_id=None, host_id=None): # pylint: disable=invalid-name + assert _sentinal is None + if core_id is not None and host_id is not None: + raise RuntimeError( + 'core_id and host_id can have only one non-None value.') + + if master is None: + return '/replica:0/task:0/device:CPU:0' + else: + # This assumes that if using more than 8 shards, + # the job configuration varies 'task'. + if core_id is not None: + host_id = core_id / 8 + return '/job:%s/task:%d/device:CPU:0' % (master, host_id) + return _placement_function + + @property + def tpu_device_placement_function(self): + master = self.master_job + job_device = '' if master is None else ('/job:%s' % master) + def _placement_function(i): + return '%s/task:%d/device:TPU:%d' % (job_device, i / 8, i % 8) + return _placement_function + + @property + def tpu_ordinal_function(self): + """Returns the TPU ordinal fn.""" + def _tpu_ordinal_function(index): + """Return the TPU ordinal associated with a shard. + + Required because the enqueue ops are placed on CPU. + + Args: + index: the shard index + + Returns: + The ordinal of the TPU device the shard's infeed should be placed on. + """ + return index % 8 + return _tpu_ordinal_function class _SIGNAL(object): @@ -269,17 +470,30 @@ class _InfeedThreadController(_InfeedOutfeedThreadBaseController): def _input_thread_fn_for_loading(self, session, enqueue_ops): count = 0 - while True: - signal = self._signal_queue.get() - if signal == _SIGNAL.STOP: - logging.info('Stop Infeed input thread.') - return - - iterations = signal - for i in range(iterations): - logging.debug('Infeed enqueue for iteration (%d, %d)', count, i) - session.run(enqueue_ops) - count += 1 + try: + while True: + signal = self._signal_queue.get() + if signal == _SIGNAL.STOP: + logging.info('Stop Infeed input thread.') + return + + if _WRAP_INPUT_FN_INTO_WHILE_LOOP: + # Enqueue batches for next loop. + session.run(enqueue_ops) + else: + iterations = signal + for i in range(iterations): + logging.debug('Infeed enqueue for iteration (%d, %d)', count, i) + session.run(enqueue_ops) + count += 1 + + except Exception: # pylint: disable=broad-except + logging.error( + 'Failed running infeed, closing session.\n' + 'You may see an exception from your main session after this.', + exc_info=1 + ) + session.close() def join(self): logging.info('Waiting for Infeed Thread to exit.') @@ -295,17 +509,16 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): dequeue. """ - def __init__(self, run_config, mode, enqueue_fn, dequeue_ops=None): - self._tpu_job = _tpu_job(run_config, mode) - self._enqueue_fn = enqueue_fn + def __init__(self, ctx, enqueue_ops, dequeue_ops=None): + self._master_job = ctx.master_job + self._enqueue_ops = enqueue_ops self._dequeue_ops = dequeue_ops def begin(self): - self._enqueue_ops = self._enqueue_fn() - self._iterations_per_loop_var = _create_iterations_per_loop() - logging.info('TPU job name %s', self._tpu_job) - self._init_op = [tpu.initialize_system(job=self._tpu_job)] - self._finalize_op = [tpu.shutdown_system(job=self._tpu_job)] + logging.info('TPU job name %s', self._master_job) + self._iterations_per_loop_var = _create_or_get_iterations_per_loop() + self._init_op = [tpu.initialize_system(job=self._master_job)] + self._finalize_op = [tpu.shutdown_system(job=self._master_job)] def after_create_session(self, session, coord): logging.info('Init TPU system') @@ -327,6 +540,7 @@ class TPUInfeedOutfeedSessionHook(session_run_hook.SessionRunHook): iterations = run_context.session.run(self._iterations_per_loop_var) self._infeed_thd_controller.send_next_batch_signal(iterations) if self._dequeue_ops is not None: + # TODO(xiejw): Refactor the outfeed dequeue into tf.while_loop. logging.info('Dequeue next batch of data from outfeed.') self._outfeed_thd_controller.send_next_batch_signal(iterations) @@ -388,7 +602,7 @@ class _TPUStopAtStepHook(session_run_hook.SessionRunHook): if self._global_step_tensor is None: raise RuntimeError('Global step should be created.') - self._iterations_per_loop_var = _create_iterations_per_loop() + self._iterations_per_loop_var = _create_or_get_iterations_per_loop() def after_create_session(self, session, coord): global_step = session.run(self._global_step_tensor) @@ -423,360 +637,288 @@ class _SetEvalIterationsHook(session_run_hook.SessionRunHook): self._num_steps = num_steps def begin(self): - self._iterations_per_loop_var = _create_iterations_per_loop() + self._iterations_per_loop_var = _create_or_get_iterations_per_loop() def after_create_session(self, session, coord): self._iterations_per_loop_var.load(self._num_steps, session=session) -class _PerShardOutput(object): - """Wraps input_fn's outputs into per-shard outputs. - - Used so that the model_fn can distinguish between sharded input and unsharded - inputs (e.g., for export_savedmodel()). - """ - - def __init__(self, output): - self.output = output - - def as_list(self): - return self.output - +def generate_per_core_enqueue_ops_fn_for_host( + ctx, input_fn, inputs_structure_recorder): + """Generates infeed enqueue ops for per-core input_fn on a single host.""" + infeed_queue_holder = {'instance': None} + + def enqueue_ops_fn(): + """A fn returns enqueue_ops.""" + num_cores_per_host = ctx.num_of_cores_per_host + per_host_sharded_inputs = [] + for core_ordinal in range(num_cores_per_host): + with ops.name_scope('ordinal_%d' % (core_ordinal)): + inputs = input_fn() + if isinstance(inputs, tuple): + features, labels = inputs + else: + features, labels = inputs, None -class _InputsHolder(object): - """A inputs holder holds the `features` and `labels' for TPU system. + inputs_structure_recorder.validate_and_record_structure( + features, labels) + flattened_inputs = ( + inputs_structure_recorder.flatten_features_and_labels( + features, labels)) + per_host_sharded_inputs.append(flattened_inputs) - Model inputs returned by the `input_fn` can have one of the following forms: + infeed_queue = tpu_feed.InfeedQueue( + number_of_tuple_elements=len(per_host_sharded_inputs[0])) + infeed_queue_holder['instance'] = infeed_queue + infeed_queue.set_configuration_from_sharded_input_tensors( + per_host_sharded_inputs) + + per_host_enqueue_ops = infeed_queue.generate_enqueue_ops( + per_host_sharded_inputs, + tpu_ordinal_function=ctx.tpu_ordinal_function) + return per_host_enqueue_ops + return enqueue_ops_fn, (lambda: infeed_queue_holder['instance']) + + +class _InputPipeline(object): + """`_InputPipeline` handles invoking `input_fn` and piping to infeed queue. + + `_InputPipeline` abstracts the per-core/per-host `input_fn` invocation from + call site. To be precise, based on the configuration in `_TPUContext`, it + invokes `input_fn` for all cores (usually multi-host TPU training) or for one + host (usually for single-host TPU evaluation), and sends all `features` and + `labels` returned by `input_fn` to TPU infeed. For per-core invocation, + `features` and `labels` are piped to infeed directly, one tuple for each + core. For per-host invocation, `features` and `labels` are split at host + (with respect to `batch_axis`) and piped to all cores accordingly. + + In addition, flatten/unflatten are handled by `_InputPipeline` also. Model + inputs returned by the `input_fn` can have one of the following forms: 1. features 2. (features, labels) Internally, form 1 is reformed to `(features, None)` as features and labels are passed separatedly to underlying methods. For TPU training, TPUEstimator - expects multiple `features` and `labels` tuples one for each shard. - - In addition, TPUEstimator allows various different structures for inputs - (namely `features` and `labels`). `features` can be `Tensor` or dict of - string name to `Tensor`, and `labels` could be `None`, `Tensor`, or dict of - string name to `Tensor`. TPU infeed/outfeed library expects flattened tensor - list. So, `features` and `labels` need to be flattened, before infeed enqueue, - and the structure of them needs to be recorded, in order to restore them after - infeed dequeue. - - `_InputsHolder` could hold the `features` and `labels` tuple for all shards - (usually multi-host TPU training) or for one host (usually for single-host TPU - evaluation), records the structure details (including presence, dict or single - tensor, dict names), validates the structure consistency cross all shards, and - encapsulates the flatten/unflatten logic. + may expect multiple `features` and `labels` tuples one for each core. + + TPUEstimator allows various different structures for inputs (namely `features` + and `labels`). `features` can be `Tensor` or dict of string name to `Tensor`, + and `labels` could be `None`, `Tensor`, or dict of string name to `Tensor`. + TPU infeed/outfeed library expects flattened tensor list. So, `features` and + `labels` need to be flattened, before infeed enqueue, and the structure of + them needs to be recorded, in order to restore them after infeed dequeue. """ - def __init__(self, features=None, labels=None, num_shards=None): - """Constructor. - - Args: - features: features for one host or a list of features one for each shard - (must be type `_PerShardOutput`). Once provided, the corresponding - `labels` should be set also and this `_InputsHolder` is frozen to - prevent from future modification. If `None`, it is expected to add - features and labels for each shard by calling `append_tuple` later. - labels: labels for one host or a list of labels one for each shard - (must be type `_PerShardOutput`). - num_shards: Number of shards in the TPU system. Must be provided unless it - can be deduced from `features`. - - Raises: - ValueError: If both `sharded_features` and `num_shards` are `None`. - """ - # Holds the features and labels for all shards. - self._feature_list = [] - self._label_list = [] - - # Holds the structure of inputs - self._feature_names = [] - self._label_names = [] - self._has_labels = False - - # Internal state. - self._initialized = False - self._frozen = False - self._sharded = False - - if features is None: - if num_shards is None: - raise ValueError( - '`features` and `num_shards` cannot be both None') - self._num_shards = num_shards - elif isinstance(features, _PerShardOutput): - self._from_sharded_inputs(features, labels, num_shards) - else: - if num_shards is None: - raise ValueError( - '`num_shards` cannot be None for unsharded features.') - self._from_unsharded_inputs(features, labels, num_shards) - - def _from_unsharded_inputs(self, features, labels, num_shards): - """Initializes the inputs with unsharded features and labels.""" - self._num_shards = num_shards - if labels is not None: - self._has_labels = True - self.append_tuple((features, labels)) - else: - self.append_tuple(features) - - self._sharded = False - self._frozen = True - - def _from_sharded_inputs(self, sharded_features, sharded_labels, num_shards): - """Initializes the inputs with sharded features and labels.""" - if not isinstance(sharded_features, _PerShardOutput): - raise ValueError('`sharded_features` must have type `_PerShardOutput`.') - features = sharded_features.as_list() - - if num_shards is not None and num_shards != len(features): - raise ValueError( - '`num_shards` should be same as the length of sharded_features.') + class InputsStructureRecorder(object): + """The recorder to record inputs structure.""" + + def __init__(self): + # Holds the structure of inputs + self._feature_names = [] + self._label_names = [] + self._has_labels = False + + # Internal state. + self._initialized = False + + def has_labels(self): + return self._has_labels + + def validate_and_record_structure(self, features, labels): + """Validates and records the structure of features` and `labels`.""" + def _extract_key_names(tensor_or_dict): + if tensor_or_dict is None: + return [] + return tensor_or_dict.keys() if isinstance(tensor_or_dict, dict) else [] + + # Extract structure. + has_labels = labels is not None + feature_names = _extract_key_names(features) + label_names = _extract_key_names(labels) + + if self._initialized: + # Verify the structure is same. The following should never happen. + assert feature_names == self._feature_names, 'feature keys mismatched' + assert label_names == self._label_names, 'label keys mismatched' + assert has_labels == self._has_labels, 'label presence mismatched' + else: + # Record structure. + self._initialized = True + self._feature_names = feature_names + self._label_names = label_names + self._has_labels = has_labels + + def flatten_features_and_labels(self, features, labels): + """Flattens the `features` and `labels` to a single tensor list.""" + flattened_inputs = [] + if self._feature_names: + # We need a fixed ordering for enqueueing and dequeueing. + flattened_inputs.extend([features[name] + for name in self._feature_names]) + else: + flattened_inputs.append(features) - self._num_shards = len(features) - if not self._num_shards: - raise ValueError('`sharded_features` should not be empty.') + if labels is not None: + if self._label_names: + # We need a fixed ordering for enqueueing and dequeueing. + flattened_inputs.extend([labels[name] for name in self._label_names]) + else: + flattened_inputs.append(labels) + return flattened_inputs + + def unflatten_features_and_labels(self, flattened_inputs): + """Restores the flattened inputs to original features and labels form. + + Args: + flattened_inputs: Flattened inputs for each shard. + + Returns: + A tuple of (`features`, `labels`), where `labels` could be None. + Each one, if present, should have identical structure (single tensor vs + dict) as the one returned by input_fn. + + Raises: + ValueError: If the number of expected tensors from `flattened_inputs` + mismatches the recorded structure. + """ + expected_num_features = (len(self._feature_names) if self._feature_names + else 1) + if self._has_labels: + expected_num_labels = (len(self._label_names) if self._label_names + else 1) + else: + expected_num_labels = 0 - if sharded_labels is not None: - if not isinstance(sharded_labels, _PerShardOutput): - raise ValueError('sharded_labels` must have type `_PerShardOutput`.') + expected_num_tensors = expected_num_features + expected_num_labels - self._has_labels = True - labels = sharded_labels.as_list() - if self._num_shards != len(labels): + if expected_num_tensors != len(flattened_inputs): raise ValueError( - 'Length of `sharded_features` and `sharded_labels` mismatch.') - - if self._has_labels: - for (f, l) in zip(features, labels): - self.append_tuple((f, l)) - else: - for f in features: - self.append_tuple(f) - - self._sharded = True - self._frozen = True - - def _extract_key_names(self, tensor_or_dict): - if tensor_or_dict is None: - return [] - - return tensor_or_dict.keys() if isinstance(tensor_or_dict, dict) else [] - - def _validate(self, features, labels): - has_labels = labels is not None - feature_names = self._extract_key_names(features) - label_names = self._extract_key_names(labels) - - if self._initialized: - self._sharded = True - # The following should never happen. - assert feature_names == self._feature_names, 'feature keys mismatched' - assert label_names == self._label_names, 'label keys mismatched' - assert has_labels == self._has_labels, 'label presence mismatched' - else: - self._initialized = True - self._feature_names = feature_names - self._label_names = label_names - self._has_labels = has_labels - - @property - def sharded(self): - if not self._frozen: - raise RuntimeError('_InputsHolder has not been frozen yet.') - return self._sharded - - @property - def num_shards(self): - if not self._frozen: - raise RuntimeError('_InputsHolder has not been frozen yet.') - return self._num_shards - - def append_tuple(self, inputs): - """Appends `inputs` for one shard into holder. - - Args: - inputs: The return from `input_fn`, which could be features or tuple of - (features, labels). After the first `inputs` appended into - `_InputsHolder`, the structure of `features` and `labels is recorded. - Any future invocation should provide the `inputs` with same structure. - - Raises: - RuntimeError: If the internal data has been frozen already. - """ - if self._frozen: - raise RuntimeError('InputsHolder has frozen, which cannot be mutated.') - - # input_fn may return either features or (features, labels) - if isinstance(inputs, tuple): - features, labels = inputs - else: - features, labels = inputs, None - - self._validate(features, labels) - - self._feature_list.append(features) - if labels is not None: - self._label_list.append(labels) - - def as_features_and_labels_tuple(self): - """Returns features and labels as grouped tuple. - - This is intended to be used to pass features and labels for all shards from - input_fn to model_fn as the parent class `Estimator` does not have the - concept of shards. So, grouped tuple is required. - - Once called, the internal data is frozen and `append_tuple` cannot be - invoked anymore. - - Returns: - A tuple of features and labels. Both have type `_PerShardOutput`, holding - the inputs for all shards. `labels` could be `None`. - - Raises: - RuntimeError: If the internal data has not been initialized. - """ - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - - assert len(self._feature_list) == self._num_shards - if not self._label_list or all(l is None for l in self._label_list): - return _PerShardOutput(self._feature_list), None - - assert len(self._label_list) == self._num_shards - return (_PerShardOutput(self._feature_list), - _PerShardOutput(self._label_list)) - - def as_sharded_flattened_inputs(self): - """Flatten the features and label as tensor lists for all shards. - - Flattened tensor list contains all tensors in `features` (dict) and `labels` - (dict). Conceptually, it has the predicated structure like: - - ```python - flatten_list = [] - for name in features: - flatten_list.append(features[name]) - for name in labels: - flatten_list.append(labels[name]) - ``` - - This method handles the label is None case and single tensor case nicely. - - Once called, the internal data is frozen and `append_tuple` cannot be - invokded anymore. - - Returns: - A list of flattened inputs one for each shard. - - Raises: - RuntimeError: If the internal data has not been initialized. - ValueError: If the inputs are sharded. - """ - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - if not self._sharded: - raise ValueError('Inputs are not sharded.') - - sharded_inputs = [] - - for shard in range(self._num_shards): - flattened_inputs = self._as_flattened_inputs( - self._feature_list[shard], - self._label_list[shard] if self._has_labels else None) - sharded_inputs.append(flattened_inputs) - - return sharded_inputs - - def as_flattened_inputs(self): - """Flatten the features and label as a single tensor list for one host.""" - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - if self._sharded: - raise ValueError('Inputs are sharded.') - - return self._as_flattened_inputs( - self._feature_list[0], - self._label_list[0] if self._has_labels else None) - - def _as_flattened_inputs(self, features, labels): - """Flattens the `features` and `labels` to a single tensor list.""" - flattened_inputs = [] - if self._feature_names: - # We need a fixed ordering for enqueueing and dequeueing. - flattened_inputs.extend([features[name] for name in self._feature_names]) - else: - flattened_inputs.append(features) - - if labels is not None: - if self._label_names: - # We need a fixed ordering for enqueueing and dequeueing. - flattened_inputs.extend([labels[name] for name in self._label_names]) + 'The number of flattened tensors mismatches expected num. ' + 'Expected {}, got {}'.format(expected_num_tensors, + len(flattened_inputs))) + if self._feature_names: + unflattened_features = dict( + zip(self._feature_names, flattened_inputs[:expected_num_features])) else: - flattened_inputs.append(labels) - return flattened_inputs + # Single tensor case + unflattened_features = flattened_inputs[0] + + if expected_num_labels == 0: + unflattened_label = None + elif self._label_names: + unflattened_label = dict(zip(self._label_names, + flattened_inputs[expected_num_features:])) + else: + # Single tensor case. + unflattened_label = flattened_inputs[expected_num_features] - def unflatten_features_and_labels(self, flattened_inputs): - """Restores the flattened inputs to original features and labels form. + return unflattened_features, unflattened_label - Once called, the internal data is frozen and `append_tuple` cannot be - invokded anymore. + def __init__(self, input_fn, batch_axis, ctx): + """Constructor. Args: - flattened_inputs: Flattened inputs for one each, which should be created - by the `as_sharded_flattened_inputs` API. - - Returns: - A tuple of (`features`, `labels`), where `labels` could be None. - Each one, if present, should have identical structure (single tensor vs - dict) as the one returned by input_fn. + input_fn: input fn for train or eval. + batch_axis: A python tuple of int values describing how each tensor + produced by the Estimator `input_fn` should be split across the TPU + compute shards. + ctx: A `_TPUContext` instance with mode. Raises: - RuntimeError: If the internal data has not been initialized. - ValueError: If the number of expected tensors from `flattened_inputs` - mismatches the recorded structure. + ValueError: If both `sharded_features` and `num_cores` are `None`. """ - self._frozen = True - if not self._initialized: - raise RuntimeError('InputsHolder has not been initialized.') - - expected_num_features = (len(self._feature_names) if self._feature_names - else 1) - if self._has_labels: - expected_num_labels = (len(self._label_names) if self._label_names - else 1) - else: - expected_num_labels = 0 + self._inputs_structure_recorder = _InputPipeline.InputsStructureRecorder() + + self._sharded_per_core = ctx.is_input_sharded_per_core() + self._input_fn = input_fn + self._infeed_queue = None + self._ctx = ctx + self._batch_axis = batch_axis + + def generate_infeed_enqueue_ops_and_dequeue_fn(self): + """Generates infeed enqueue ops and dequeue_fn.""" + # While tf.while_loop is called, the body function, which invokes + # `enqueue_fn` passed in, is called to construct the graph. So, input_fn + # structure is recorded. + enqueue_ops = self._invoke_input_fn_and_record_structure() + + def dequeue_fn(): + """dequeue_fn is used by TPU to retrieve the tensors.""" + values = self._infeed_queue.generate_dequeue_op() + # The unflatten process uses the structure information recorded above. + return self._inputs_structure_recorder.unflatten_features_and_labels( + values) + + return (enqueue_ops, dequeue_fn) + + def _invoke_input_fn_and_record_structure(self): + if self._sharded_per_core: + # Per-Core input pipeline deployment. + tpu_host_placement_fn = self._ctx.tpu_host_placement_function + enqueue_ops = [] + infeed_queues = [] + + # Invoke input pipeline for each core and placed on the corresponding + # host. + num_hosts = self._ctx.num_hosts + for host_id in range(num_hosts): + host_device = tpu_host_placement_fn(host_id=host_id) + with ops.device(host_device): + with ops.name_scope('input_pipeline_task%d' % (host_id)): + enqueue_ops_fn, infeed_queue_getter = ( + generate_per_core_enqueue_ops_fn_for_host( + self._ctx, self._input_fn, self._inputs_structure_recorder)) + + if _WRAP_INPUT_FN_INTO_WHILE_LOOP: + enqueue_ops.append(_wrap_computation_in_while_loop( + device=host_device, op_fn=enqueue_ops_fn)) + else: + enqueue_ops.append(enqueue_ops_fn()) + # Infeed_queue_getter must be called after enqueue_ops_fn is called. + infeed_queues.append(infeed_queue_getter()) + + # infeed_queue is used to generate dequeue ops. The only thing it uses for + # dequeue is dtypes and types. So, any one can be used. Here, grab the + # first one. + self._infeed_queue = infeed_queues[0] + return enqueue_ops - expected_num_tensors = expected_num_features + expected_num_labels - - if expected_num_tensors != len(flattened_inputs): - raise ValueError( - 'The number of flattened tensors mismatches expected num. ' - 'Expected {}, got {}'.format(expected_num_tensors, - len(flattened_inputs))) - if self._feature_names: - unflattened_features = dict(zip(self._feature_names, - flattened_inputs[:expected_num_features])) - else: - # Single tensor case - unflattened_features = flattened_inputs[0] - - if expected_num_labels == 0: - unflattened_label = None - elif self._label_names: - unflattened_label = dict(zip(self._label_names, - flattened_inputs[expected_num_features:])) else: - # Single tensor case. - unflattened_label = flattened_inputs[expected_num_features] - - return unflattened_features, unflattened_label + # TODO(b/67051042): Extend this to multi-host support. + host_id = 0 + host_device = self._ctx.tpu_host_placement_function(host_id=host_id) + def enqueue_fn(): + with ops.device(host_device): + with ops.name_scope('input_pipeline_task%d' % (host_id)): + inputs = self._input_fn() + if isinstance(inputs, tuple): + features, labels = inputs + else: + features, labels = inputs, None + self._inputs_structure_recorder.validate_and_record_structure( + features, labels) + unsharded_tensor_list = ( + self._inputs_structure_recorder.flatten_features_and_labels( + features, labels)) + + self._infeed_queue = tpu_feed.InfeedQueue( + tuple_types=[t.dtype for t in unsharded_tensor_list], + tuple_shapes=[t.shape for t in unsharded_tensor_list], + shard_dimensions=self._batch_axis) + self._infeed_queue.set_number_of_shards(self._ctx.num_cores) + + def placement_fn(core_id): + return self._ctx.tpu_host_placement_function(core_id=core_id) + return ( + self._infeed_queue.split_inputs_and_generate_enqueue_ops( + unsharded_tensor_list, + placement_function=placement_fn)) + + if _WRAP_INPUT_FN_INTO_WHILE_LOOP: + return _wrap_computation_in_while_loop(device=host_device, + op_fn=enqueue_fn) + else: + return enqueue_fn() class _ModelFnWrapper(object): @@ -789,20 +931,17 @@ class _ModelFnWrapper(object): train and eval step. """ - def __init__(self, model_fn, config, params, mode, train_batch_size, - eval_batch_size): + def __init__(self, model_fn, config, params, ctx): self._model_fn = model_fn self._config = config self._params = params - self._mode = mode - self._train_batch_size = train_batch_size - self._eval_batch_size = eval_batch_size + self._ctx = ctx def call_without_tpu(self, features, labels): # Let CrossShardOptimizer be called without TPU in model_fn, since it's # common to set the train_op even when running evaluate() or predict(). with tpu_function.tpu_shard_context(1): - return self._call_model_fn(features, labels, use_tpu=False) + return self._call_model_fn(features, labels) def convert_to_single_tpu_train_step(self, dequeue_fn): """Converts user provided model_fn` as a single train step on TPU. @@ -832,7 +971,7 @@ class _ModelFnWrapper(object): features, labels = dequeue_fn() estimator_spec = self._verify_estimator_spec( - self._call_model_fn(features, labels, use_tpu=True)) + self._call_model_fn(features, labels)) loss, train_op = estimator_spec.loss, estimator_spec.train_op with ops.control_dependencies([train_op]): return array_ops.identity(loss) @@ -864,13 +1003,13 @@ class _ModelFnWrapper(object): A tuple of eval_fn and eval_metrics. The eval_fn representing the eval step for TPU. and eval_metrics is an `_EvalMetrics` instance. """ - eval_metrics = _EvalMetrics() + eval_metrics = _EvalMetrics(self._ctx) def eval_step(total_loss): """Evaluation step function for use inside a while loop.""" features, labels = dequeue_fn() - tpu_estimator_spec = self._call_model_fn(features, labels, use_tpu=True) + tpu_estimator_spec = self._call_model_fn(features, labels) if not isinstance(tpu_estimator_spec, TPUEstimatorSpec): raise RuntimeError( 'estimator_spec used by TPU evaluation must have type' @@ -884,11 +1023,7 @@ class _ModelFnWrapper(object): return math_ops.add(total_loss, loss) return eval_step, eval_metrics - @property - def config(self): - return self._config - - def _call_model_fn(self, features, labels, use_tpu): + def _call_model_fn(self, features, labels): """Calls the model_fn with required parameters.""" model_fn_args = util.fn_args(self._model_fn) kwargs = {} @@ -899,12 +1034,11 @@ class _ModelFnWrapper(object): if 'labels' in model_fn_args: kwargs['labels'] = labels - else: - if labels is not None: - raise ValueError( - 'model_fn does not take labels, but input_fn returns labels.') + elif labels is not None: + raise ValueError( + 'model_fn does not take labels, but input_fn returns labels.') if 'mode' in model_fn_args: - kwargs['mode'] = self._mode + kwargs['mode'] = self._ctx.mode if 'config' in model_fn_args: kwargs['config'] = config if 'params' in model_fn_args: @@ -915,16 +1049,16 @@ class _ModelFnWrapper(object): 'model_fn ({}) does not include params argument, ' 'required by TPUEstimator to pass batch size as ' 'params[\'batch_size\']'.format(self._model_fn)) - if self._mode == model_fn_lib.ModeKeys.TRAIN: - params[_BATCH_SIZE_KEY] = _per_shard_batch_size( - self._train_batch_size, config, use_tpu) - elif (self._mode == model_fn_lib.ModeKeys.EVAL and - self._eval_batch_size is not None): - params[_BATCH_SIZE_KEY] = _per_shard_batch_size( - self._eval_batch_size, config, use_tpu) + + batch_size_for_model_fn = self._ctx.batch_size_for_model_fn + if batch_size_for_model_fn is not None: + params[_BATCH_SIZE_KEY] = batch_size_for_model_fn estimator_spec = self._model_fn(features=features, **kwargs) - if (not use_tpu) and isinstance(estimator_spec, TPUEstimatorSpec): + if (self._ctx.is_running_on_cpu() and + isinstance(estimator_spec, TPUEstimatorSpec)): + # The estimator_spec will be passed to `Estimator` directly, which expects + # type `EstimatorSpec`. return estimator_spec.as_estimator_spec() else: return estimator_spec @@ -947,7 +1081,8 @@ class _ModelFnWrapper(object): class _EvalMetrics(object): """Class wraps TPUEstimator.eval_metrics.""" - def __init__(self): + def __init__(self, ctx): + self._ctx = ctx self._metric_fn = None self._is_dict = False self._tensor_keys = [] @@ -971,8 +1106,6 @@ class _EvalMetrics(object): if isinstance(eval_metrics[1], (tuple, list)): fn_args = util.fn_args(eval_metrics[0]) - if 'self' in fn_args: - fn_args = tuple([arg for arg in fn_args if arg != 'self']) if len(eval_metrics[1]) != len(fn_args): raise RuntimeError( 'In TPUEstimatorSpec.eval_metrics, length of tensors does not ' @@ -1030,7 +1163,7 @@ class _EvalMetrics(object): raise RuntimeError('Eval metrics have not been recorded yet') return self._tensors - def to_metric_metric_ops_for_tpu(self, run_config, dummy_update_op): + def to_metric_metric_ops_for_tpu(self, dummy_update_op): """Creates the eval_metric_ops now based on the TPU outfeed. `eval_metric_ops` is defined in `EstimatorSpec`. From all shards, tensors @@ -1039,7 +1172,6 @@ class _EvalMetrics(object): metric fn. Args: - run_config: A `RunConfig` instance. dummy_update_op: A dummy update op. Returns: @@ -1051,9 +1183,7 @@ class _EvalMetrics(object): RuntimeError: If outfeed tensor is scalar. """ - num_shards = run_config.tpu_config.num_shards - job = _tpu_job(run_config, model_fn_lib.ModeKeys.EVAL) - job_device = '' if job is None else ('/job:%s' % job) + num_cores = self._ctx.num_cores # For each i, dequeue_ops[i] is a list containing the tensors from all # shards. This list is concatenated later. @@ -1062,8 +1192,9 @@ class _EvalMetrics(object): dequeue_ops.append([]) # Outfeed ops execute on each JF node. - for i in xrange(num_shards): - with ops.device('%s/task:%d/device:TPU:%d' % (job_device, i / 8, i % 8)): + tpu_device_placement_fn = self._ctx.tpu_device_placement_function + for i in xrange(num_cores): + with ops.device(tpu_device_placement_fn(i)): outfeed_tensors = tpu_ops.outfeed_dequeue_tuple( dtypes=self._tensor_dtypes, shapes=self._tensor_shapes) for j, item in enumerate(outfeed_tensors): @@ -1071,7 +1202,7 @@ class _EvalMetrics(object): # It is assumed evaluation always happends on single host TPU system. So, # place all ops on tpu host if possible. - with ops.device('{}/device:CPU:0'.format(job_device)): + with ops.device(self._ctx.tpu_host_placement_function(core_id=0)): for i, item in enumerate(dequeue_ops): if dequeue_ops[i][0].shape.ndims == 0: raise RuntimeError( @@ -1116,9 +1247,9 @@ class TPUEstimator(estimator_lib.Estimator): specify `train_batch_size` in constructor, and then get the batch size for each shard in `input_fn` and `model_fn` by `params['batch_size']`. If `TPUConfig.per_host_input_for_training` is `True`, `input_fn` is invoked per - host rather than per shard. In this case, a global batch size is transformed a + host rather than per core. In this case, a global batch size is transformed a per-host batch size in params for `input_fn`, but `model_fn` still gets - per-shard batch size. + per-core batch size. For evaluation, if `eval_batch_size` is None, it is executed on CPU, even if `use_tpu` is `True`. If `eval_batch_size` is not `None`, it is executed on @@ -1276,9 +1407,7 @@ class TPUEstimator(estimator_lib.Estimator): # We cannot store config and params in this constructor as parent # constructor might change them, such as assigning a temp dir for # config.model_dir. - model_function = _augment_model_fn(model_fn, train_batch_size, - eval_batch_size, use_tpu, - batch_axis) + model_function = self._augment_model_fn(model_fn, batch_axis) # Passing non-None params as wrapped model_fn has it. params = params or {} @@ -1287,12 +1416,13 @@ class TPUEstimator(estimator_lib.Estimator): model_dir=model_dir, config=config, params=params) - self._use_tpu = use_tpu - self._train_batch_size = train_batch_size - self._eval_batch_size = eval_batch_size self._iterations_per_training_loop = ( self._config.tpu_config.iterations_per_loop) + # All properties passed to _TPUContext are immutable. + self._ctx = _TPUContext(self._config, train_batch_size, eval_batch_size, + use_tpu) + def _create_global_step(self, graph): """Creates a global step suitable for TPUs. @@ -1308,10 +1438,10 @@ class TPUEstimator(estimator_lib.Estimator): return _create_global_step(graph) def _convert_train_steps_to_hooks(self, steps, max_steps): - if _is_running_on_cpu(self._use_tpu, model_fn_lib.ModeKeys.TRAIN, - self._eval_batch_size): - return super(TPUEstimator, self)._convert_train_steps_to_hooks( - steps, max_steps) + with self._ctx.with_mode(model_fn_lib.ModeKeys.TRAIN) as ctx: + if ctx.is_running_on_cpu(): + return super(TPUEstimator, self)._convert_train_steps_to_hooks( + steps, max_steps) # On TPU. if steps is None and max_steps is None: @@ -1329,9 +1459,9 @@ class TPUEstimator(estimator_lib.Estimator): steps, max_steps)] def _convert_eval_steps_to_hooks(self, steps): - if _is_running_on_cpu(self._use_tpu, model_fn_lib.ModeKeys.EVAL, - self._eval_batch_size): - return super(TPUEstimator, self)._convert_eval_steps_to_hooks(steps) + with self._ctx.with_mode(model_fn_lib.ModeKeys.EVAL) as ctx: + if ctx.is_running_on_cpu(): + return super(TPUEstimator, self)._convert_eval_steps_to_hooks(steps) if steps is None: raise ValueError('Evaluate `steps` must be set on TPU. Cannot be `None`.') @@ -1371,197 +1501,115 @@ class TPUEstimator(estimator_lib.Estimator): if 'config' in input_fn_args: kwargs['config'] = config - # Setting the batch size in params first. This helps user to have same - # input_fn for use_tpu=True/False. - if mode == model_fn_lib.ModeKeys.TRAIN: - kwargs['params'][_BATCH_SIZE_KEY] = ( - _per_shard_batch_size(self._train_batch_size, config, self._use_tpu) - if not config.tpu_config.per_host_input_for_training else - self._train_batch_size) - elif (mode == model_fn_lib.ModeKeys.EVAL and - self._eval_batch_size is not None): - # For TPU evaluation, input_fn is invoked for one host (instead of shard). - kwargs['params'][_BATCH_SIZE_KEY] = self._eval_batch_size - - if _is_running_on_cpu(self._use_tpu, mode, self._eval_batch_size): - with ops.device('/device:CPU:0'): - return input_fn(**kwargs) + with self._ctx.with_mode(mode) as ctx: + # Setting the batch size in params first. This helps user to have same + # input_fn for use_tpu=True/False. + batch_size_for_input_fn = ctx.batch_size_for_input_fn + if batch_size_for_input_fn is not None: + kwargs['params'][_BATCH_SIZE_KEY] = batch_size_for_input_fn - job = _tpu_job(config, mode) - def placement_function(index): - if job is None: - return '/replica:0/task:0/device:CPU:0' - else: - return '/job:%s/task:%d/device:CPU:0' % (job, index / 8) - - if mode == model_fn_lib.ModeKeys.TRAIN: - if not config.tpu_config.per_host_input_for_training: - # Now for TPU training. - num_shards = config.tpu_config.num_shards - inputs = _InputsHolder(num_shards=num_shards) - for i in range(config.tpu_config.num_shards): - with ops.device(placement_function(i)): - inputs.append_tuple(input_fn(**kwargs)) - return inputs.as_features_and_labels_tuple() - else: - # TODO(xiejw): Extend this to multi-host support. - with ops.device(placement_function(0)): + if ctx.is_running_on_cpu(): + with ops.device('/device:CPU:0'): return input_fn(**kwargs) - # Now for TPU evaluation. - with ops.device(placement_function(0)): - return input_fn(**kwargs) - - -# TODO(b/64607814): Ensure batch_axis works with nested structures. -def _create_infeed_enqueue_ops_and_dequeue_fn(inputs_holder, run_config, - batch_axis, mode): - """Utility to convert input_fn to enqueue and dequeue fns for TPU. - - Args: - inputs_holder: An `_InputsHolder` holding features and labels. - run_config: A `RunConfig` instance. - batch_axis: A python list of batch dimensions. - mode: ModeKeys - - Returns: - A tuple of (dequeue_fn, enqueue_fn) - """ - if inputs_holder.sharded: - sharded_inputs = inputs_holder.as_sharded_flattened_inputs() - - infeed_queue = tpu_feed.InfeedQueue( - number_of_tuple_elements=len(sharded_inputs[0])) - infeed_queue.set_configuration_from_sharded_input_tensors(sharded_inputs) - else: - unsharded_inputs = inputs_holder.as_flattened_inputs() - infeed_queue = tpu_feed.InfeedQueue( - tuple_types=[t.dtype for t in unsharded_inputs], - tuple_shapes=[t.shape for t in unsharded_inputs], - shard_dimensions=batch_axis) - infeed_queue.set_number_of_shards(inputs_holder.num_shards) - - def dequeue_fn(): - """dequeue_fn is used by the train_step in TPU to retrieve the tensors.""" - values = infeed_queue.generate_dequeue_op() - return inputs_holder.unflatten_features_and_labels(values) - - def tpu_ordinal_function(index): - """Return the TPU ordinal associated with a shard. - - Required because the enqueue ops are placed on CPU. - - Args: - index: the shard index - - Returns: - The ordinal of the TPU device the shard's infeed should be placed on. - """ - return index % 8 - - def enqueue_fn(): - """enqueue_fn is used to add ops to the graph to send tensors.""" - if inputs_holder.sharded: - return infeed_queue.generate_enqueue_ops( - sharded_inputs, tpu_ordinal_function=tpu_ordinal_function) - else: - job = _tpu_job(run_config, mode) - def placement_function(index): - if job is None: - return '/replica:0/task:0/device:CPU:0' - else: - # This assumes that if using more than 8 shards, - # the job configuration varies 'task'. - return '/job:%s/task:%d/device:CPU:0' % (job, index / 8) - return infeed_queue.split_inputs_and_generate_enqueue_ops( - unsharded_inputs, placement_function=placement_function) - - return (dequeue_fn, enqueue_fn) - - -def _augment_model_fn(model_fn, train_batch_size, eval_batch_size, use_tpu, - batch_axis): - """Returns a new model_fn, which wraps the TPU support.""" - - def _model_fn(features, labels, mode, config, params): - """A Estimator `model_fn` for TPUEstimator.""" - model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, mode, - train_batch_size, eval_batch_size) - - # TODO(jhseu): Move to PREDICT to TPU. - if _is_running_on_cpu(use_tpu, mode, eval_batch_size): - logging.info('Running %s on CPU', mode) - return model_fn_wrapper.call_without_tpu(features, labels) - - inputs = _InputsHolder(features=features, labels=labels, - num_shards=config.tpu_config.num_shards) - - dequeue_fn, enqueue_fn = _create_infeed_enqueue_ops_and_dequeue_fn( - inputs, config, batch_axis, mode) - - if mode == model_fn_lib.ModeKeys.TRAIN: - loss = _train_on_tpu_system(model_fn_wrapper, dequeue_fn) - hooks = [ - TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn), - training.LoggingTensorHook( - {'loss': array_ops.identity(loss), - 'step': training.get_global_step()}, - every_n_secs=30) - ] - summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss) - with ops.control_dependencies([loss]): - update_ops = _sync_variables_ops() - - # Validate the TPU training graph to catch basic errors - _validate_tpu_training_graph() - - return model_fn_lib.EstimatorSpec( - mode, - loss=loss, - training_hooks=hooks, - train_op=control_flow_ops.group(*update_ops)) - - # Now eval. - total_loss, eval_metric_ops = _eval_on_tpu_system( - model_fn_wrapper, dequeue_fn) - iterations_per_loop_var = _create_iterations_per_loop() - mean_loss = math_ops.div( - total_loss, - math_ops.cast(iterations_per_loop_var, dtype=total_loss.dtype)) - - # Creates a dummy metric update_op for all metrics. Estimator expects all - # metrics in eval_metric_ops have update_op and calls them one by one. The - # real metric update_ops are invoked in a separated thread. So, here give - # Estimator the dummy op for all metrics. - with ops.control_dependencies([mean_loss]): - # After TPU evaluation computation is done (the mean_loss tensor), reads - # all variables back from TPU and updates the eval step counter properly. - internal_ops_to_run = _sync_variables_ops() - internal_ops_to_run.append( - _increase_eval_step_op(iterations_per_loop_var)) - with ops.control_dependencies(internal_ops_to_run): - dummy_update_op = control_flow_ops.no_op() - - eval_metric_ops, eval_update_ops = ( - eval_metric_ops.to_metric_metric_ops_for_tpu( - config, dummy_update_op)) - hooks = [ - TPUInfeedOutfeedSessionHook(config, mode, enqueue_fn, eval_update_ops), - ] - - return model_fn_lib.EstimatorSpec( - mode, - loss=mean_loss, - evaluation_hooks=hooks, - eval_metric_ops=eval_metric_ops) - return _model_fn - - -def _eval_on_tpu_system(model_fn_wrapper, dequeue_fn): + # For TPU computation, input_fn should be invoked in a tf.while_loop for + # performance. While constructing the tf.while_loop, the structure of + # inputs returned by the `input_fn` needs to be recorded. The structure + # includes whether features or labels is dict or single Tensor, dict keys, + # tensor shapes, and dtypes. The recorded structure is used to create the + # infeed dequeue ops, which must be wrapped and passed as a Fn, called + # inside the TPU computation, as the TPU computation is wrapped inside a + # tf.while_loop also. So, we either pass input_fn to model_fn or pass + # dequeue_fn to model_fn. Here, `input_fn` is passed directly as + # `features` in `model_fn` signature. + def _input_fn(): + return input_fn(**kwargs) + return _input_fn + + def _augment_model_fn(self, model_fn, batch_axis): + """Returns a new model_fn, which wraps the TPU support.""" + + def _model_fn(features, labels, mode, config, params): + """A Estimator `model_fn` for TPUEstimator.""" + with self._ctx.with_mode(mode) as ctx: + model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, ctx) + + # TODO(jhseu): Move to PREDICT to TPU. + if ctx.is_running_on_cpu(): + logging.info('Running %s on CPU', mode) + return model_fn_wrapper.call_without_tpu(features, labels) + + assert labels is None, '`labels` passed to `model_fn` must be `None`.' + # TPUEstimator._call_input_fn passes `input_fn` as features to here. + assert callable(features), '`input_fn` is not callable.' + input_fn = features + + input_holders = _InputPipeline(input_fn, batch_axis, ctx) + enqueue_ops, dequeue_fn = ( + input_holders.generate_infeed_enqueue_ops_and_dequeue_fn()) + + if mode == model_fn_lib.ModeKeys.TRAIN: + loss = _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn) + hooks = [ + TPUInfeedOutfeedSessionHook(ctx, enqueue_ops), + training.LoggingTensorHook( + {'loss': array_ops.identity(loss), + 'step': training.get_global_step()}, + every_n_secs=30) + ] + summary.scalar(model_fn_lib.LOSS_METRIC_KEY, loss) + with ops.control_dependencies([loss]): + update_ops = _sync_variables_ops() + + # Validate the TPU training graph to catch basic errors + _validate_tpu_training_graph() + + return model_fn_lib.EstimatorSpec( + mode, + loss=loss, + training_hooks=hooks, + train_op=control_flow_ops.group(*update_ops)) + + # Now eval. + total_loss, eval_metric_ops = _eval_on_tpu_system( + ctx, model_fn_wrapper, dequeue_fn) + iterations_per_loop_var = _create_or_get_iterations_per_loop() + mean_loss = math_ops.div( + total_loss, + math_ops.cast(iterations_per_loop_var, dtype=total_loss.dtype)) + + # Creates a dummy metric update_op for all metrics. Estimator expects + # all metrics in eval_metric_ops have update_op and calls them one by + # one. The real metric update_ops are invoked in a separated thread. So, + # here give Estimator the dummy op for all metrics. + with ops.control_dependencies([mean_loss]): + # After TPU evaluation computation is done (the mean_loss tensor), + # reads all variables back from TPU and updates the eval step counter + # properly + internal_ops_to_run = _sync_variables_ops() + internal_ops_to_run.append( + _increase_eval_step_op(iterations_per_loop_var)) + with ops.control_dependencies(internal_ops_to_run): + dummy_update_op = control_flow_ops.no_op() + + eval_metric_ops, eval_update_ops = ( + eval_metric_ops.to_metric_metric_ops_for_tpu(dummy_update_op)) + hooks = [ + TPUInfeedOutfeedSessionHook(ctx, enqueue_ops, eval_update_ops), + ] + + return model_fn_lib.EstimatorSpec( + mode, + loss=mean_loss, + evaluation_hooks=hooks, + eval_metric_ops=eval_metric_ops) + return _model_fn + + +def _eval_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): """Executes `model_fn_wrapper` multiple times on all TPU shards.""" - config = model_fn_wrapper.config.tpu_config - num_shards = config.num_shards - iterations_per_loop_var = _create_iterations_per_loop() + num_cores = ctx.num_cores + iterations_per_loop_var = _create_or_get_iterations_per_loop() single_tpu_eval_step, eval_metric_ops = ( model_fn_wrapper.convert_to_single_tpu_eval_step(dequeue_fn)) @@ -1574,15 +1622,15 @@ def _eval_on_tpu_system(model_fn_wrapper, dequeue_fn): (loss,) = tpu.shard(multi_tpu_eval_steps_on_single_shard, inputs=[], - num_shards=num_shards, + num_shards=num_cores, outputs_from_all_shards=False) return loss, eval_metric_ops -def _train_on_tpu_system(model_fn_wrapper, dequeue_fn): +def _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn): """Executes `model_fn_wrapper` multiple times on all TPU shards.""" - num_shards = model_fn_wrapper.config.tpu_config.num_shards - iterations_per_loop_var = _create_iterations_per_loop() + num_cores = ctx.num_cores + iterations_per_loop_var = _create_or_get_iterations_per_loop() single_tpu_train_step = model_fn_wrapper.convert_to_single_tpu_train_step( dequeue_fn) @@ -1596,11 +1644,27 @@ def _train_on_tpu_system(model_fn_wrapper, dequeue_fn): (loss,) = tpu.shard(multi_tpu_train_steps_on_single_shard, inputs=[], - num_shards=num_shards, + num_shards=num_cores, outputs_from_all_shards=False) return loss +def _wrap_computation_in_while_loop(device, op_fn): + """Wraps the ops generated by `op_fn` in tf.while_loop.""" + def computation(i): + with ops.control_dependencies(op_fn()): + return i + 1 + + iterations_per_loop_var = _create_or_get_iterations_per_loop() + # By setting parallel_iterations=1, the parallel execution in while_loop is + # basically turned off. + with ops.device(device): + iterations = array_ops.identity(iterations_per_loop_var) + return control_flow_ops.while_loop( + lambda i: i < iterations, + computation, [constant_op.constant(0)], parallel_iterations=1) + + def _validate_tpu_training_graph(): """Validate graph before running distributed training. diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD index 80a5debe996a330d64e62ce430d33d4111ee8767..6139c1d5838c24414549b4e2bc4722175f2d1925 100644 --- a/tensorflow/contrib/training/BUILD +++ b/tensorflow/contrib/training/BUILD @@ -26,6 +26,7 @@ py_library( "python/training/resample.py", "python/training/sampling_ops.py", "python/training/sequence_queueing_state_saver.py", + "python/training/sgdr_learning_rate_decay.py", "python/training/training.py", "python/training/tuner.py", ], @@ -41,6 +42,7 @@ py_library( "//tensorflow/python:data_flow_ops", "//tensorflow/python:errors", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:layers_base", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", @@ -111,6 +113,7 @@ py_test( "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", + "//tensorflow/python:sparse_ops", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", "//tensorflow/python:training", @@ -125,9 +128,12 @@ py_test( srcs = ["python/training/feeding_queue_runner_test.py"], srcs_version = "PY2AND3", deps = [ - ":training_py", "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_test_lib", + "//tensorflow/python:framework_ops", + "//tensorflow/python:session", + "//tensorflow/python:training", + "//tensorflow/python/estimator:inputs_queues", + "//third_party/py/numpy", ], ) @@ -139,7 +145,6 @@ py_test( deps = [ ":training_py", "//tensorflow/python:client_testlib", - "@six_archive//:six", ], ) @@ -243,12 +248,12 @@ py_test( "//tensorflow/contrib/metrics:metrics_py", "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", - "//tensorflow/python:client", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:random_seed", + "//tensorflow/python:session", "//tensorflow/python:state_ops", "//tensorflow/python:summary", "//tensorflow/python:training", @@ -270,6 +275,7 @@ py_test( "//tensorflow/contrib/layers:layers_py", "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:gradients", "//tensorflow/python:math_ops", "//tensorflow/python:platform", "//tensorflow/python:random_seed", diff --git a/tensorflow/contrib/training/python/training/bucket_ops.py b/tensorflow/contrib/training/python/training/bucket_ops.py index 5523cc375fc20dc167fee0eaa6f1682dc1892c3f..95fbc50cba73b25b748c31ecd443eb19c0b6fc8a 100644 --- a/tensorflow/contrib/training/python/training/bucket_ops.py +++ b/tensorflow/contrib/training/python/training/bucket_ops.py @@ -31,6 +31,7 @@ from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util +from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops @@ -47,7 +48,6 @@ _dtypes = input_py._dtypes _store_sparse_tensors = input_py._store_sparse_tensors _validate_keep_input = input_py._validate_keep_input _shapes = input_py._shapes -_smart_cond = input_py._smart_cond _which_queue = input_py._which_queue # pylint: enable=protected-access @@ -239,7 +239,7 @@ def bucket(tensors, ] return control_flow_ops.group(*enqueues, name="group_enqueues") - maybe_enqueue = _smart_cond( + maybe_enqueue = utils.smart_cond( keep_input, enqueue_which, control_flow_ops.no_op) diff --git a/tensorflow/contrib/training/python/training/hparam.py b/tensorflow/contrib/training/python/training/hparam.py index c95a73ce4492caa81cc6b902a782717de06c1b63..391899b34f90be25e10450ebf4e285ed2d39446f 100644 --- a/tensorflow/contrib/training/python/training/hparam.py +++ b/tensorflow/contrib/training/python/training/hparam.py @@ -25,6 +25,7 @@ import six from tensorflow.contrib.training.python.training import hparam_pb2 from tensorflow.python.framework import ops from tensorflow.python.util import compat +from tensorflow.python.util import deprecation # Define the regular expression for parsing a single clause of the input # (delimited by commas). A legal clause looks like: @@ -470,24 +471,29 @@ class HParams(object): type_map[name] = param_type values_map = parse_values(values, type_map) - return self.set_from_map(values_map) + return self.override_from_dict(values_map) - def set_from_map(self, values_map): + def override_from_dict(self, values_dict): """Override hyperparameter values, parsing new values from a dictionary. Args: - values_map: Dictionary of name:value pairs. + values_dict: Dictionary of name:value pairs. Returns: The `HParams` instance. Raises: - ValueError: If `values_map` cannot be parsed. + ValueError: If `values_dict` cannot be parsed. """ - for name, value in values_map.items(): + for name, value in values_dict.items(): self.set_hparam(name, value) return self + @deprecation.deprecated(None, 'Use `override_from_dict`.') + def set_from_map(self, values_map): + """DEPRECATED. Use override_from_dict.""" + return self.override_from_dict(values_dict=values_map) + def set_model_structure(self, model_structure): self._model_structure = model_structure @@ -515,7 +521,7 @@ class HParams(object): ValueError: If `values_json` cannot be parsed. """ values_map = json.loads(values_json) - return self.set_from_map(values_map) + return self.override_from_dict(values_map) def values(self): """Return the hyperparameter values as a Python dictionary. @@ -526,6 +532,9 @@ class HParams(object): """ return {n: getattr(self, n) for n in self._hparam_types.keys()} + def __contains__(self, key): + return key in self._hparam_types + def __str__(self): return str(sorted(self.values().items())) diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py index b01116a2139f76bab2e6219048c7c1aec013e626..f54514cefd39cab93e5c3a34786a6bb751b97704 100644 --- a/tensorflow/contrib/training/python/training/hparam_test.py +++ b/tensorflow/contrib/training/python/training/hparam_test.py @@ -32,6 +32,11 @@ class HParamsTest(test.TestCase): with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'): hparams.parse('xyz=123') + def testContains(self): + hparams = hparam.HParams(foo=1) + self.assertTrue('foo' in hparams) + self.assertFalse('bar' in hparams) + def testSomeValues(self): hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6') self.assertDictEqual({'aaa': 1, 'b': 2.0, 'c_c': 'relu6'}, hparams.values()) @@ -93,11 +98,11 @@ class HParamsTest(test.TestCase): def testSetFromMap(self): hparams = hparam.HParams(a=1, b=2.0, c='tanh') - hparams.set_from_map({'a': -2, 'c': 'identity'}) + hparams.override_from_dict({'a': -2, 'c': 'identity'}) self.assertDictEqual({'a': -2, 'c': 'identity', 'b': 2.0}, hparams.values()) hparams = hparam.HParams(x=1, b=2.0, d=[0.5]) - hparams.set_from_map({'d': [0.1, 0.2, 0.3]}) + hparams.override_from_dict({'d': [0.1, 0.2, 0.3]}) self.assertDictEqual({'d': [0.1, 0.2, 0.3], 'x': 1, 'b': 2.0}, hparams.values()) diff --git a/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc b/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc index a1fbea57dd1202c1a22e6b3570e9378555fe3498..cff765d1e832e5a593462283444d7c4ed7831636 100644 --- a/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc +++ b/tensorflow/contrib/verbs/grpc_verbs_service_impl.cc @@ -43,21 +43,21 @@ VerbsService::Stub::Stub( const std::shared_ptr< ::grpc::ChannelInterface>& channel) : channel_(channel), rpcmethod_GetRemoteAddress_(grpcVerbsService_method_names[0], - ::grpc::internal::RpcMethod::NORMAL_RPC, + ::grpc::RpcMethod::NORMAL_RPC, channel) {} ::grpc::Status VerbsService::Stub::GetRemoteAddress( ::grpc::ClientContext* context, const GetRemoteAddressRequest& request, GetRemoteAddressResponse* response) { - return ::grpc::internal::BlockingUnaryCall( + return ::grpc::BlockingUnaryCall( channel_.get(), rpcmethod_GetRemoteAddress_, context, request, response); } VerbsService::AsyncService::AsyncService() { for (int i = 0; i < 1; ++i) { - AddMethod(new ::grpc::internal::RpcServiceMethod( + AddMethod(new ::grpc::RpcServiceMethod( grpcVerbsService_method_names[i], - ::grpc::internal::RpcMethod::NORMAL_RPC, + ::grpc::RpcMethod::NORMAL_RPC, nullptr)); ::grpc::Service::MarkMethodAsync(i); } diff --git a/tensorflow/contrib/verbs/grpc_verbs_service_impl.h b/tensorflow/contrib/verbs/grpc_verbs_service_impl.h index 86431ca030c38c56155801202714ee4a49b764df..6e2bf86dac2aa84ff453aaefbfc57cd3ee8bc1fd 100644 --- a/tensorflow/contrib/verbs/grpc_verbs_service_impl.h +++ b/tensorflow/contrib/verbs/grpc_verbs_service_impl.h @@ -28,6 +28,15 @@ limitations under the License. #include "tensorflow/contrib/verbs/verbs_service.pb.h" namespace grpc { + +// ensure internal namespace exists +namespace internal { +// bring in contents of external namespace +using namespace ::grpc; +} // namespace internal +// bring in contents of internal namespace +using namespace internal; + class CompletionQueue; class Channel; class RpcService; @@ -61,7 +70,7 @@ class VerbsService GRPC_FINAL { private: std::shared_ptr< ::grpc::ChannelInterface> channel_; - const ::grpc::internal::RpcMethod rpcmethod_GetRemoteAddress_; + const ::grpc::RpcMethod rpcmethod_GetRemoteAddress_; }; static std::unique_ptr NewStub( const std::shared_ptr< ::grpc::ChannelInterface>& channel, diff --git a/tensorflow/contrib/xla_tf_graph/BUILD b/tensorflow/contrib/xla_tf_graph/BUILD deleted file mode 100644 index 4a3a2de9b5e58cfab2e6f8de5c6789f1cbcebde7..0000000000000000000000000000000000000000 --- a/tensorflow/contrib/xla_tf_graph/BUILD +++ /dev/null @@ -1,67 +0,0 @@ -# Description: -# contains parts of TensorFlow that are experimental or unstable and which are not supported. - -package( - default_visibility = ["//visibility:public"], -) - -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -load("//tensorflow:tensorflow.bzl", "tf_cc_test") - -filegroup( - name = "all_files", - srcs = glob( - ["**/*"], - exclude = [ - "**/METADATA", - "**/OWNERS", - ], - ), -) - -cc_library( - name = "xla_tf_graph_util", - srcs = [ - "xla_tf_graph_util.cc", - ], - hdrs = [ - "xla_tf_graph_util.h", - ], - deps = [ - "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/xla:status_macros", - "//tensorflow/compiler/xla/client", - "//tensorflow/compiler/xla/client:client_library", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], -) - -tf_cc_test( - name = "xla_tf_graph_util_test", - srcs = ["xla_tf_graph_util_test.cc"], - linkstatic = 1, - tags = ["nomac"], # b/63908145 - deps = [ - ":xla_tf_graph_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/cc:function_ops", - "//tensorflow/cc:scope", - "//tensorflow/compiler/jit:xla_cpu_jit", - "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/compiler/xla/client:client_library", - "//tensorflow/compiler/xla/service:hlo_module_config", - "//tensorflow/core:core_cpu_internal", - "//tensorflow/core:framework_internal", - "//tensorflow/core:ops", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core/kernels:cwise_op", - ], -) diff --git a/tensorflow/contrib/xla_tf_graph/README.md b/tensorflow/contrib/xla_tf_graph/README.md deleted file mode 100644 index a374189e813107bcf3fe71032d4baf16b3d164a2..0000000000000000000000000000000000000000 --- a/tensorflow/contrib/xla_tf_graph/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# Xla Tf Graph - -## Description - -This module contains utilities to treat xla representation as tf graph to support mobile SOC experiments and leverage tf tools. - -Maintainers: -- Satoshi Kataoka (satok@google.com, github.com/satok16) diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc deleted file mode 100644 index 302aa6457ab08a30bca9c28a5f162331111c4b77..0000000000000000000000000000000000000000 --- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.cc +++ /dev/null @@ -1,247 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h" - -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/client_library.h" -#include "tensorflow/core/platform/protobuf.h" - -namespace tensorflow { -namespace xla_tf_graph { - -namespace { - -constexpr const char* const GRAPH_NAME = "xla_tf_graph"; -constexpr const char* const NODE_NAME_PREFIX = "xla"; - -Status ConvertPrimitiveTypeToDataType(const xla::PrimitiveType p_type, - DataType* d_type) { - switch (p_type) { - case xla::PRED: - *d_type = DT_BOOL; - return Status::OK(); - case xla::S8: - *d_type = DT_INT8; - return Status::OK(); - case xla::S16: - *d_type = DT_INT16; - return Status::OK(); - case xla::S32: - *d_type = DT_INT32; - return Status::OK(); - case xla::S64: - *d_type = DT_INT64; - return Status::OK(); - case xla::U8: - *d_type = DT_UINT8; - return Status::OK(); - case xla::U16: - *d_type = DT_UINT16; - return Status::OK(); - case xla::F16: - *d_type = DT_HALF; - return Status::OK(); - case xla::F32: - *d_type = DT_FLOAT; - return Status::OK(); - case xla::F64: - *d_type = DT_DOUBLE; - return Status::OK(); - default: - return errors::InvalidArgument( - "Unsupported PrimitiveType in ConvertPrimitiveTypeToDataType ", - xla::PrimitiveType_Name(p_type)); - } -} - -Status ConvertXlaShapeToTensorShapeType(const xla::Shape& xla_shape, - std::vector* tensor_shapes, - std::vector* data_types) { - switch (xla_shape.element_type()) { - case xla::TUPLE: { - for (const xla::Shape& element_shape : xla_shape.tuple_shapes()) { - if (element_shape.element_type() == xla::TUPLE) { - return errors::InvalidArgument("Nested tuple is not allowed."); - } - TF_RETURN_IF_ERROR(ConvertXlaShapeToTensorShapeType( - element_shape, tensor_shapes, data_types)); - } - return Status::OK(); - } - case xla::PRED: - case xla::S8: - case xla::S16: - case xla::S32: - case xla::S64: - case xla::U8: - case xla::U16: - case xla::U32: - case xla::U64: - case xla::F16: - case xla::F32: - case xla::F64: { - TensorShape shape; - DataType type; - TF_RETURN_IF_ERROR( - ConvertPrimitiveTypeToDataType(xla_shape.element_type(), &type)); - for (const int64& dim : xla_shape.dimensions()) { - shape.AddDim(dim); - } - tensor_shapes->emplace_back(shape); - data_types->emplace_back(type); - return Status::OK(); - } - default: - return errors::InvalidArgument( - "Unsupported PrimitiveType in ConvertXlaShapeToTensorShapeType ", - xla::PrimitiveType_Name(xla_shape.element_type())); - } -} - -string BuildXlaNodeName(const xla::OperationRequest& operation_request, - const string& xla_op_type, const string& suffix) { - const string name = strings::StrCat( - NODE_NAME_PREFIX, "/", operation_request.output_handle().handle(), "/", - xla_op_type); - if (suffix.empty()) { - return name; - } else { - return strings::StrCat(name, "/", suffix); - } -} - -string BuildXlaNodeName(const xla::OperationRequest& operation_request, - const string& xla_op_type) { - return BuildXlaNodeName(operation_request, xla_op_type, ""); -} - -string BuildXlaNodeOp(const protobuf::Message& msg, const string& suffix) { - return strings::StrCat(msg.GetDescriptor()->name(), "/", suffix); -} - -string BuildXlaNodeOp(const protobuf::Message& msg) { - return BuildXlaNodeOp(msg, ""); -} - -Status ConvertOpRequestToXlaNode(const xla::OperationRequest& operation_request, - XlaNode* xla_node) { - const xla::OpRequest& op_request = operation_request.request(); - switch (op_request.op_case()) { - case xla::OpRequest::kBinaryOpRequest: { - const xla::BinaryOpRequest& op = op_request.binary_op_request(); - xla_node->op_type = - BuildXlaNodeOp(op, xla::BinaryOperation_Name(op.binop())); - xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type); - xla_node->input_ids.emplace_back(std::make_tuple(op.lhs().handle(), 0)); - xla_node->input_ids.emplace_back(std::make_tuple(op.rhs().handle(), 0)); - for (const int64& dim : op.broadcast_dimensions()) { - xla_node->broadcast_dimensions.emplace_back(dim); - } - break; - } - case xla::OpRequest::kParameterRequest: { - const xla::ParameterRequest& op = op_request.parameter_request(); - xla_node->op_type = BuildXlaNodeOp(op, ""); - xla_node->name = - BuildXlaNodeName(operation_request, xla_node->op_type, op.name()); - break; - } - case xla::OpRequest::kVariadicOpRequest: { - const xla::VariadicOpRequest& op = op_request.variadic_op_request(); - xla_node->op_type = - BuildXlaNodeOp(op, xla::VariadicOperation_Name(op.varop())); - xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type); - for (const xla::ComputationDataHandle& handle : op.operands()) { - xla_node->input_ids.emplace_back(std::make_tuple(handle.handle(), 0)); - } - break; - } - case xla::OpRequest::kGetTupleElementRequest: { - const xla::GetTupleElementRequest& op = - op_request.get_tuple_element_request(); - xla_node->op_type = BuildXlaNodeOp(op); - xla_node->name = BuildXlaNodeName(operation_request, xla_node->op_type); - xla_node->input_ids.emplace_back( - std::make_tuple(op.operand().handle(), op.index())); - break; - } - default: - // TODO(satok): Implement all possible cases. - LOG(FATAL) << "Op request: " << op_request.op_case() - << " is not supported yet."; - break; - } - - CHECK(!xla_node->name.empty()); - CHECK(!xla_node->op_type.empty()); - - TF_RETURN_IF_ERROR(ConvertXlaShapeToTensorShapeType( - operation_request.output_shape(), &xla_node->output_shapes, - &xla_node->output_data_types)); - return Status::OK(); -} - -void SetupXlaCpuClient(std::unique_ptr* flib_def, - std::unique_ptr* compiler) { - xla::Client* client = xla::ClientLibrary::LocalClientOrDie(); - XlaOpRegistry::RegisterCompilationKernels(); - - FunctionDefLibrary flib; - flib_def->reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib)); - - // Setup compiler options - XlaCompiler::Options options; - DeviceType device_type(DEVICE_CPU_XLA_JIT); - options.device_type = &device_type; - options.flib_def = flib_def->get(); - options.client = client; - compiler->reset(new XlaCompiler(options)); -} - -} // namespace - -xla::StatusOr> -ConvertTfGraphToXlaSessionModule(const std::vector& args, - std::unique_ptr graph) { - CHECK(graph); - - std::unique_ptr flib_def; - std::unique_ptr compiler; - - SetupXlaCpuClient(&flib_def, &compiler); - - // Compile graph and build computation - XlaCompiler::CompilationResult result; - TF_CHECK_OK(compiler->CompileGraph(XlaCompiler::CompileOptions(), GRAPH_NAME, - std::move(graph), args, &result)); - - return result.computation->Snapshot(); -} - -xla::StatusOr> -ConvertXlaSessionModuleToXlaNodes(const xla::SessionModule& session_module) { - std::unordered_map xla_nodes; - for (const auto& operation_request : session_module.entry().requests()) { - XlaNode xla_node; - TF_RETURN_IF_ERROR( - ConvertOpRequestToXlaNode(operation_request.second, &xla_node)); - xla_nodes.emplace(operation_request.first, xla_node); - } - return std::move(xla_nodes); -} - -} // namespace xla_tf_graph -} // namespace tensorflow diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h deleted file mode 100644 index e635290851f7e5d078d98d845e7488fc3cd94049..0000000000000000000000000000000000000000 --- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h +++ /dev/null @@ -1,72 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_ -#define TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_ - -#include - -#include "tensorflow/compiler/tf2xla/xla_compiler.h" -#include "tensorflow/compiler/xla/client/client.h" -#include "tensorflow/compiler/xla/status_macros.h" -#include "tensorflow/core/framework/function.h" -#include "tensorflow/core/graph/graph.h" -#include "tensorflow/core/platform/macros.h" - -namespace tensorflow { -namespace xla_tf_graph { - -// A set of utilities to handle xla computation requests. -// These utilities help developers leverage existing tools to work with -// xla computations, also provide a way to support TensorFlow ops by -// implementing xla computations so that they can do experiments on their -// specialized environments. - -// A structure to represent typed attributes of TensorFlow graph node. -// This structure contains op specific attributes as members so that -// we can treat them explicitly. -struct XlaNode { - // Unique node name - string name; - // Op type of xla computation - string op_type; - // List of pair of unique id and port of input node. - // We store this value instead - // of node name in order not to wait for all XlaNodes to be constructed. - std::vector> input_ids; - // Oputput shapes - std::vector output_shapes; - // Output data types - std::vector output_data_types; - - //--------------------------- - // Op specific attributes - // #xla::OpRequest::kBinaryOpRequest - std::vector broadcast_dimensions; -}; - -// Convert a tf graph to a xla session module -xla::StatusOr> -ConvertTfGraphToXlaSessionModule(const std::vector& args, - std::unique_ptr graph); - -// Convert a xla session module to a map to XlaNode from unique id -xla::StatusOr> -ConvertXlaSessionModuleToXlaNodes(const xla::SessionModule& session_module); - -} // namespace xla_tf_graph -} // namespace tensorflow - -#endif // TENSORFLOW_CONTRIB_XLA_TF_GRAPH_XLA_TF_GRAPH_UTIL_H_ diff --git a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc b/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc deleted file mode 100644 index 144269303ee140bb7a9a30133a5d88b41b4f4273..0000000000000000000000000000000000000000 --- a/tensorflow/contrib/xla_tf_graph/xla_tf_graph_util_test.cc +++ /dev/null @@ -1,134 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/contrib/xla_tf_graph/xla_tf_graph_util.h" -#include "tensorflow/cc/framework/scope.h" -#include "tensorflow/cc/ops/function_ops.h" -#include "tensorflow/cc/ops/standard_ops.h" -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/client_library.h" -#include "tensorflow/compiler/xla/service/hlo_module_config.h" -#include "tensorflow/compiler/xla/shape_util.h" -#include "tensorflow/core/platform/test.h" - -namespace tensorflow { -namespace xla_tf_graph { - -static std::unique_ptr BuildAddGraph() { - Scope scope = Scope::NewRootScope().ExitOnError(); - auto a = ops::_Arg(scope.WithOpName("A"), DT_INT32, 0); - auto b = ops::_Arg(scope.WithOpName("B"), DT_INT32, 1); - // See tf2xla/kernels/binary_ops.cc - auto c = ops::Add(scope.WithOpName("C"), a, b); - auto d = ops::_Retval(scope.WithOpName("D"), c, 0); - std::unique_ptr graph(new Graph(OpRegistry::Global())); - TF_CHECK_OK(scope.ToGraph(graph.get())); - return graph; -} - -static std::vector BuildAddGraphArguments() { - // Builds a description of the arguments. - std::vector args(2); - args[0].kind = XlaCompiler::Argument::kParameter; - args[0].type = DT_INT32; - // Difference of dimension will add extra broadcast_dimensions. - // broadcast_dimension generates an additional HloInstruction - // in user_computation.cc - args[0].shape = xla::ShapeUtil::MakeShape(xla::S32, {2, 2}); - args[1].kind = XlaCompiler::Argument::kParameter; - args[1].type = DT_INT32; - args[1].shape = xla::ShapeUtil::MakeShape(xla::S32, {2}); - return args; -} - -// CAVEAT: Debug purpose only. -// This function dumps a protobuf string format of HloModule. -static void DumpHloGraphForDebug(const std::vector& args, - std::unique_ptr graph) { - std::unique_ptr flib_def; - std::unique_ptr flr; - std::unique_ptr compiler; - - xla::Client* client = xla::ClientLibrary::LocalClientOrDie(); - XlaOpRegistry::RegisterCompilationKernels(); - - FunctionDefLibrary flib; - flib_def.reset(new FunctionLibraryDefinition(OpRegistry::Global(), flib)); - - // Compiles the graph. - XlaCompiler::Options options; - DeviceType device_type("XLA_CPU_JIT"); - options.device_type = &device_type; - options.client = client; - options.flib_def = flib_def.get(); - compiler.reset(new XlaCompiler(options)); - - // Compile graph - XlaCompiler::CompilationResult result; - TF_CHECK_OK(compiler->CompileGraph(XlaCompiler::CompileOptions(), "dump", - std::move(graph), args, &result)); - - // Convert to hlo - xla::Computation& computation = *result.computation; - - xla::Service* service( - static_cast(xla::ClientLibrary::GetXlaService( - static_cast(client)->platform()))); - const xla::ComputationTracker& computation_tracker = - service->computation_tracker(); - - auto user_computation_status = - computation_tracker.Resolve(computation.handle()); - TF_CHECK_OK(user_computation_status.status()); - auto user_computation = user_computation_status.ConsumeValueOrDie(); - xla::VersionedComputationHandle versioned_handle = - user_computation->GetVersionedHandle(); - std::unique_ptr hlo_module = - std::move(computation_tracker - .BuildHloModule(versioned_handle, xla::HloModuleConfig()) - .ValueOrDie()); - VLOG(1) << "--- DUMP HLO ---"; - VLOG(1) << hlo_module->ToString(); -} - -TEST(XlaTfGraphUtil, ConvertTfGraphToSessionModule) { - // Builds a description of the arguments. - std::vector args = BuildAddGraphArguments(); - std::unique_ptr graph = BuildAddGraph(); - - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr session_module, - ConvertTfGraphToXlaSessionModule(args, std::move(graph))); - - ASSERT_EQ(4, session_module->entry().requests_size()); - - VLOG(1) << "--- DUMP ---"; - VLOG(1) << session_module->DebugString(); - DumpHloGraphForDebug(args, BuildAddGraph()); -} - -TEST(XlaTfGraphUtil, ConvertXlaSessionModuleToXlaNodes) { - std::vector args = BuildAddGraphArguments(); - std::unique_ptr graph = BuildAddGraph(); - TF_ASSERT_OK_AND_ASSIGN( - std::unique_ptr session_module, - ConvertTfGraphToXlaSessionModule(args, std::move(graph))); - TF_ASSERT_OK_AND_ASSIGN(auto xla_nodes, - ConvertXlaSessionModuleToXlaNodes(*session_module)); - EXPECT_EQ(session_module->entry().requests_size(), xla_nodes.size()); -} - -} // namespace xla_tf_graph -} // namespace tensorflow diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index f60c0d76cf8a06583df7371a7be01d755b97a883..1c58aa3315bb88eeb69035c11f56ddfd3d651eee 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -163,6 +163,7 @@ CORE_PROTO_SRCS = [ "framework/function.proto", "framework/graph.proto", "framework/graph_transfer_info.proto", + "framework/iterator.proto", "framework/kernel_def.proto", "framework/log_memory.proto", "framework/node_def.proto", @@ -510,6 +511,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":lib", + ":lib_internal", ":op_gen_overrides_proto_cc", ":protos_all_cc", ], @@ -781,6 +783,7 @@ cc_library( "//tensorflow/core/kernels:dataset_ops", "//tensorflow/core/kernels:fake_quant_ops", "//tensorflow/core/kernels:function_ops", + "//tensorflow/core/kernels:histogram_op", "//tensorflow/core/kernels:image", "//tensorflow/core/kernels:io", "//tensorflow/core/kernels:linalg", @@ -1398,6 +1401,7 @@ LIB_INTERNAL_PUBLIC_HEADERS = tf_additional_lib_hdrs() + [ "platform/platform.h", "platform/protobuf_internal.h", "platform/setround.h", + "platform/snappy.h", "platform/tensor_coding.h", "platform/tracing.h", ] @@ -1408,7 +1412,7 @@ cc_library( hdrs = LIB_INTERNAL_PUBLIC_HEADERS, copts = tf_copts(), defines = tf_additional_lib_defines() + [ - "SNAPPY", + "TF_USE_SNAPPY", ] + tf_additional_verbs_lib_defines() + tf_additional_mpi_lib_defines() + tf_additional_gdr_lib_defines(), @@ -2176,6 +2180,7 @@ tf_cuda_library( ":lib", ":lib_internal", ":protos_all_cc", + ":stream_executor", "//third_party/eigen3", ] + if_static([":gpu_runtime_impl"]), ) @@ -2258,7 +2263,6 @@ cc_library( "lib/io/block_builder.h", "lib/io/format.h", "lib/random/philox_random_test_utils.h", - "platform/snappy.h", ], deps = [ ":lib", @@ -2505,6 +2509,7 @@ tf_cc_test( srcs = ["framework/op_gen_lib_test.cc"], deps = [ ":op_gen_lib", + ":protos_all_cc", ":test", ":test_main", ], @@ -3354,6 +3359,11 @@ filegroup( data = glob(["api_def/base_api/*"]), ) +filegroup( + name = "python_api_def", + data = glob(["api_def/python_api/*"]), +) + tf_cc_test( name = "api_test", srcs = ["api_def/api_test.cc"], diff --git a/tensorflow/core/api_def/python_api/api_def_A.pbtxt b/tensorflow/core/api_def/python_api/api_def_A.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..df9b3ad0b69235eaf22c1b84b624e4037084547d --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_A.pbtxt @@ -0,0 +1,56 @@ +op { + graph_op_name: "Abs" + visibility: HIDDEN +} +op { + graph_op_name: "AddManySparseToTensorsMap" + visibility: HIDDEN +} +op { + graph_op_name: "AddN" + visibility: HIDDEN +} +op { + graph_op_name: "AddSparseToTensorsMap" + visibility: HIDDEN +} +op { + graph_op_name: "AdjustContrastv2" + visibility: HIDDEN +} +op { + graph_op_name: "All" + visibility: HIDDEN +} +op { + graph_op_name: "AllCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "Any" + visibility: HIDDEN +} +op { + graph_op_name: "Assert" + visibility: HIDDEN +} +op { + graph_op_name: "AudioSummary" + visibility: HIDDEN +} +op { + graph_op_name: "AudioSummaryV2" + visibility: HIDDEN +} +op { + graph_op_name: "AvgPool" + visibility: HIDDEN +} +op { + graph_op_name: "AvgPool3DGrad" + visibility: HIDDEN +} +op { + graph_op_name: "AvgPoolGrad" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_B.pbtxt b/tensorflow/core/api_def/python_api/api_def_B.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..49c74ccad28f8e1ecc12b5ad0ce6a054670da36a --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_B.pbtxt @@ -0,0 +1,142 @@ +op { + graph_op_name: "Barrier" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierClose" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierIncompleteSize" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierInsertMany" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierReadySize" + visibility: HIDDEN +} +op { + graph_op_name: "BarrierTakeMany" + visibility: HIDDEN +} +op { + graph_op_name: "BatchCholesky" + visibility: HIDDEN +} +op { + graph_op_name: "BatchCholeskyGrad" + visibility: HIDDEN +} +op { + graph_op_name: "BatchFFT" + visibility: HIDDEN +} +op { + graph_op_name: "BatchFFT2D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchFFT3D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchIFFT" + visibility: HIDDEN +} +op { + graph_op_name: "BatchIFFT2D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchIFFT3D" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatMul" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixDeterminant" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixInverse" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixSolve" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixSolveLs" + visibility: HIDDEN +} +op { + graph_op_name: "BatchMatrixTriangularSolve" + visibility: HIDDEN +} +op { + graph_op_name: "BatchNormWithGlobalNormalization" + visibility: HIDDEN +} +op { + graph_op_name: "BatchNormWithGlobalNormalizationGrad" + visibility: HIDDEN +} +op { + graph_op_name: "BatchSelfAdjointEig" + visibility: HIDDEN +} +op { + graph_op_name: "BatchSelfAdjointEigV2" + visibility: HIDDEN +} +op { + graph_op_name: "BatchSvd" + visibility: HIDDEN +} +op { + graph_op_name: "BatchToSpace" + visibility: HIDDEN +} +op { + graph_op_name: "BiasAdd" + visibility: HIDDEN +} +op { + graph_op_name: "BiasAddV1" + visibility: HIDDEN +} +op { + graph_op_name: "BitwiseAnd" + endpoint { + name: "bitwise.bitwise_and" + } +} +op { + graph_op_name: "BitwiseOr" + endpoint { + name: "bitwise.bitwise_or" + } +} +op { + graph_op_name: "BitwiseXor" + endpoint { + name: "bitwise.bitwise_xor" + } +} +op { + graph_op_name: "BroadcastArgs" + visibility: HIDDEN +} +op { + graph_op_name: "BroadcastGradientArgs" + visibility: HIDDEN +} +op { + graph_op_name: "Bucketize" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_C.pbtxt b/tensorflow/core/api_def/python_api/api_def_C.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..42ed24b1336efb59d835c87980f032adde59344a --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_C.pbtxt @@ -0,0 +1,59 @@ +op { + graph_op_name: "CTCBeamSearchDecoder" + visibility: HIDDEN +} +op { + graph_op_name: "CTCGreedyDecoder" + visibility: HIDDEN +} +op { + graph_op_name: "CTCLoss" + visibility: HIDDEN +} +op { + graph_op_name: "Cholesky" + endpoint { + name: "cholesky" + } + endpoint { + name: "linalg.cholesky" + } +} +op { + graph_op_name: "Complex" + visibility: HIDDEN +} +op { + graph_op_name: "ComplexAbs" + visibility: HIDDEN +} +op { + graph_op_name: "ComputeAccidentalHits" + visibility: HIDDEN +} +op { + graph_op_name: "Concat" + visibility: HIDDEN +} +op { + graph_op_name: "ConcatOffset" + visibility: HIDDEN +} +op { + graph_op_name: "ConcatV2" + visibility: HIDDEN +} +op { + graph_op_name: "Conj" + visibility: HIDDEN +} +op { + graph_op_name: "Const" + visibility: HIDDEN +} +op { + graph_op_name: "CropAndResize" + endpoint { + name: "image.crop_and_resize" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_D.pbtxt b/tensorflow/core/api_def/python_api/api_def_D.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..c73982aed0cd718f65645f248e6dd16115d948c5 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_D.pbtxt @@ -0,0 +1,74 @@ +op { + graph_op_name: "DebugGradientIdentity" + visibility: HIDDEN +} +op { + graph_op_name: "DecodeAndCropJpeg" + endpoint { + name: "image.decode_and_crop_jpeg" + } +} +op { + graph_op_name: "DecodeBmp" + endpoint { + name: "image.decode_bmp" + } +} +op { + graph_op_name: "DecodeCSV" + visibility: HIDDEN +} +op { + graph_op_name: "DecodeGif" + endpoint { + name: "image.decode_gif" + } +} +op { + graph_op_name: "DecodeJpeg" + endpoint { + name: "image.decode_jpeg" + } +} +op { + graph_op_name: "DecodePng" + endpoint { + name: "image.decode_png" + } +} +op { + graph_op_name: "DeleteSessionTensor" + visibility: HIDDEN +} +op { + graph_op_name: "DepthwiseConv2dNative" + endpoint { + name: "nn.depthwise_conv2d_native" + } +} +op { + graph_op_name: "DepthwiseConv2dNativeBackpropFilter" + endpoint { + name: "nn.depthwise_conv2d_native_backprop_filter" + } +} +op { + graph_op_name: "DepthwiseConv2dNativeBackpropInput" + endpoint { + name: "nn.depthwise_conv2d_native_backprop_input" + } +} +op { + graph_op_name: "DeserializeManySparse" + visibility: HIDDEN +} +op { + graph_op_name: "DestroyTemporaryVariable" + visibility: HIDDEN +} +op { + graph_op_name: "DrawBoundingBoxes" + endpoint { + name: "image.draw_bounding_boxes" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_E.pbtxt b/tensorflow/core/api_def/python_api/api_def_E.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..236c344167a825a3476bb2a51534eee19bc3d138 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_E.pbtxt @@ -0,0 +1,46 @@ +op { + graph_op_name: "EditDistance" + visibility: HIDDEN +} +op { + graph_op_name: "Elu" + endpoint { + name: "nn.elu" + } +} +op { + graph_op_name: "EluGrad" + visibility: HIDDEN +} +op { + graph_op_name: "EncodeJpeg" + endpoint { + name: "image.encode_jpeg" + } +} +op { + graph_op_name: "EncodePng" + endpoint { + name: "image.encode_png" + } +} +op { + graph_op_name: "Exit" + visibility: HIDDEN +} +op { + graph_op_name: "ExpandDims" + visibility: HIDDEN +} +op { + graph_op_name: "ExtractGlimpse" + endpoint { + name: "image.extract_glimpse" + } +} +op { + graph_op_name: "ExtractJpegShape" + endpoint { + name: "image.extract_jpeg_shape" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_F.pbtxt b/tensorflow/core/api_def/python_api/api_def_F.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..a29b6a372513b8e463563212291d655c2e501615 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_F.pbtxt @@ -0,0 +1,73 @@ +op { + graph_op_name: "FFT" + endpoint { + name: "fft" + } + endpoint { + name: "spectral.fft" + } +} +op { + graph_op_name: "FIFOQueue" + visibility: HIDDEN +} +op { + graph_op_name: "FIFOQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "Fact" + visibility: HIDDEN +} +op { + graph_op_name: "FakeQueue" + visibility: HIDDEN +} +op { + graph_op_name: "FixedLengthRecordReader" + visibility: HIDDEN +} +op { + graph_op_name: "FixedLengthRecordReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "FixedUnigramCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "FloorDiv" + visibility: HIDDEN +} +op { + graph_op_name: "FloorMod" + visibility: HIDDEN +} +op { + graph_op_name: "FractionalAvgPool" + endpoint { + name: "nn.fractional_avg_pool" + } +} +op { + graph_op_name: "FractionalAvgPoolGrad" + visibility: HIDDEN +} +op { + graph_op_name: "FractionalMaxPool" + endpoint { + name: "nn.fractional_max_pool" + } +} +op { + graph_op_name: "FractionalMaxPoolGrad" + visibility: HIDDEN +} +op { + graph_op_name: "FusedBatchNorm" + visibility: HIDDEN +} +op { + graph_op_name: "FusedBatchNormV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_G.pbtxt b/tensorflow/core/api_def/python_api/api_def_G.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..8235d245feb5403600532cfd05456e256b3faf0d --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_G.pbtxt @@ -0,0 +1,16 @@ +op { + graph_op_name: "GenerateVocabRemapping" + visibility: HIDDEN +} +op { + graph_op_name: "GetSessionHandle" + visibility: HIDDEN +} +op { + graph_op_name: "GetSessionHandleV2" + visibility: HIDDEN +} +op { + graph_op_name: "GetSessionTensor" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_H.pbtxt b/tensorflow/core/api_def/python_api/api_def_H.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..9f3fe2eb08a384e5e74018b8089d8fc6293deb03 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_H.pbtxt @@ -0,0 +1,18 @@ +op { + graph_op_name: "HSVToRGB" + endpoint { + name: "image.hsv_to_rgb" + } +} +op { + graph_op_name: "HashTable" + visibility: HIDDEN +} +op { + graph_op_name: "HashTableV2" + visibility: HIDDEN +} +op { + graph_op_name: "HistogramSummary" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_I.pbtxt b/tensorflow/core/api_def/python_api/api_def_I.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..db6a54dbd43030c433c5716cefe2fe410694031a --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_I.pbtxt @@ -0,0 +1,55 @@ +op { + graph_op_name: "IFFT" + endpoint { + name: "ifft" + } + endpoint { + name: "spectral.ifft" + } +} +op { + graph_op_name: "IdentityReader" + visibility: HIDDEN +} +op { + graph_op_name: "IdentityReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "ImageSummary" + visibility: HIDDEN +} +op { + graph_op_name: "InTopK" + visibility: HIDDEN +} +op { + graph_op_name: "InTopKV2" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTable" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTableFromTextFile" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTableFromTextFileV2" + visibility: HIDDEN +} +op { + graph_op_name: "InitializeTableV2" + visibility: HIDDEN +} +op { + graph_op_name: "InvGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Invert" + endpoint { + name: "bitwise.invert" + } +} diff --git a/tensorflow/core/api_def/python_api/api_def_L.pbtxt b/tensorflow/core/api_def/python_api/api_def_L.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..083fbdae6f5706745ce763a23d4aaec25ca51b3c --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_L.pbtxt @@ -0,0 +1,96 @@ +op { + graph_op_name: "L2Loss" + endpoint { + name: "nn.l2_loss" + } +} +op { + graph_op_name: "LMDBReader" + visibility: HIDDEN +} +op { + graph_op_name: "LRN" + endpoint { + name: "nn.local_response_normalization" + } + endpoint { + name: "nn.lrn" + } +} +op { + graph_op_name: "LRNGrad" + visibility: HIDDEN +} +op { + graph_op_name: "LearnedUnigramCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "LinSpace" + endpoint { + name: "lin_space" + } + endpoint { + name: "linspace" + } +} +op { + graph_op_name: "ListDiff" + visibility: HIDDEN +} +op { + graph_op_name: "LoadAndRemapMatrix" + visibility: HIDDEN +} +op { + graph_op_name: "LogMatrixDeterminant" + visibility: HIDDEN +} +op { + graph_op_name: "LogSoftmax" + visibility: HIDDEN +} +op { + graph_op_name: "LogUniformCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableExport" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableExportV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableFind" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableFindV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableImport" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableImportV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableInsert" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableInsertV2" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableSize" + visibility: HIDDEN +} +op { + graph_op_name: "LookupTableSizeV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_M.pbtxt b/tensorflow/core/api_def/python_api/api_def_M.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..c8840e0c09009992a51b4ca08fb0fc3e97868ec6 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_M.pbtxt @@ -0,0 +1,174 @@ +op { + graph_op_name: "MatMul" + visibility: HIDDEN +} +op { + graph_op_name: "MatrixBandPart" + endpoint { + name: "linalg.band_part" + } + endpoint { + name: "matrix_band_part" + } +} +op { + graph_op_name: "MatrixDeterminant" + endpoint { + name: "linalg.det" + } + endpoint { + name: "matrix_determinant" + } +} +op { + graph_op_name: "MatrixDiag" + endpoint { + name: "linalg.diag" + } + endpoint { + name: "matrix_diag" + } +} +op { + graph_op_name: "MatrixDiagPart" + endpoint { + name: "linalg.diag_part" + } + endpoint { + name: "matrix_diag_part" + } +} +op { + graph_op_name: "MatrixInverse" + endpoint { + name: "linalg.inv" + } + endpoint { + name: "matrix_inverse" + } +} +op { + graph_op_name: "MatrixSetDiag" + endpoint { + name: "linalg.set_diag" + } + endpoint { + name: "matrix_set_diag" + } +} +op { + graph_op_name: "MatrixSolve" + endpoint { + name: "linalg.solve" + } + endpoint { + name: "matrix_solve" + } +} +op { + graph_op_name: "MatrixSolveLs" + visibility: HIDDEN +} +op { + graph_op_name: "MatrixTriangularSolve" + endpoint { + name: "linalg.triangular_solve" + } + endpoint { + name: "matrix_triangular_solve" + } +} +op { + graph_op_name: "Max" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPool" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPool3DGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPool3DGradGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGradGrad" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGradGradWithArgmax" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolGradWithArgmax" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolV2" + visibility: HIDDEN +} +op { + graph_op_name: "MaxPoolWithArgmax" + endpoint { + name: "nn.max_pool_with_argmax" + } +} +op { + graph_op_name: "Mean" + visibility: HIDDEN +} +op { + graph_op_name: "Merge" + visibility: HIDDEN +} +op { + graph_op_name: "MergeSummary" + visibility: HIDDEN +} +op { + graph_op_name: "Min" + visibility: HIDDEN +} +op { + graph_op_name: "MirrorPad" + visibility: HIDDEN +} +op { + graph_op_name: "MirrorPadGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Mul" + visibility: HIDDEN +} +op { + graph_op_name: "MutableDenseHashTable" + visibility: HIDDEN +} +op { + graph_op_name: "MutableDenseHashTableV2" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTable" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTableOfTensors" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTableOfTensorsV2" + visibility: HIDDEN +} +op { + graph_op_name: "MutableHashTableV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_N.pbtxt b/tensorflow/core/api_def/python_api/api_def_N.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..60da4dcafe886bafa44301cdf3c375551530fdea --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_N.pbtxt @@ -0,0 +1,16 @@ +op { + graph_op_name: "Neg" + visibility: HIDDEN +} +op { + graph_op_name: "NegTrain" + visibility: HIDDEN +} +op { + graph_op_name: "NonMaxSuppression" + visibility: HIDDEN +} +op { + graph_op_name: "NonMaxSuppressionV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_O.pbtxt b/tensorflow/core/api_def/python_api/api_def_O.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..3a9f0f40321eede5f67a71e651a32d3cb485bae4 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_O.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "OneHot" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_P.pbtxt b/tensorflow/core/api_def/python_api/api_def_P.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..87ca53e0b9a49e50f1937ff077b0129343ad7c42 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_P.pbtxt @@ -0,0 +1,68 @@ +op { + graph_op_name: "Pack" + visibility: HIDDEN +} +op { + graph_op_name: "Pad" + visibility: HIDDEN +} +op { + graph_op_name: "PadV2" + visibility: HIDDEN +} +op { + graph_op_name: "PaddingFIFOQueue" + visibility: HIDDEN +} +op { + graph_op_name: "PaddingFIFOQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "ParallelConcat" + visibility: HIDDEN +} +op { + graph_op_name: "ParameterizedTruncatedNormal" + visibility: HIDDEN +} +op { + graph_op_name: "ParseExample" + visibility: HIDDEN +} +op { + graph_op_name: "ParseSingleSequenceExample" + visibility: HIDDEN +} +op { + graph_op_name: "Placeholder" + visibility: HIDDEN +} +op { + graph_op_name: "Pow" + visibility: HIDDEN +} +op { + graph_op_name: "Print" + visibility: HIDDEN +} +op { + graph_op_name: "PriorityQueue" + visibility: HIDDEN +} +op { + graph_op_name: "PriorityQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "Prod" + visibility: HIDDEN +} +op { + graph_op_name: "PyFunc" + visibility: HIDDEN +} +op { + graph_op_name: "PyFuncStateless" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_Q.pbtxt b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..0dfb5bb703bba7cb7576f48b112d3014d69c2824 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_Q.pbtxt @@ -0,0 +1,83 @@ +op { + graph_op_name: "Qr" + endpoint { + name: "linalg.qr" + } + endpoint { + name: "qr" + } +} +op { + graph_op_name: "QuantizedAvgPool" + endpoint { + name: "nn.quantized_avg_pool" + } +} +op { + graph_op_name: "QuantizedMaxPool" + endpoint { + name: "nn.quantized_max_pool" + } +} +op { + graph_op_name: "QuantizedReluX" + endpoint { + name: "nn.quantized_relu_x" + } +} +op { + graph_op_name: "QueueClose" + visibility: HIDDEN +} +op { + graph_op_name: "QueueCloseV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeue" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueMany" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueManyV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueUpTo" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueUpToV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueDequeueV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueue" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueueMany" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueueManyV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueEnqueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "QueueSize" + visibility: HIDDEN +} +op { + graph_op_name: "QueueSizeV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_R.pbtxt b/tensorflow/core/api_def/python_api/api_def_R.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..0c8a8a4d4235ebe86a45d66d32dc307329d3f5ed --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_R.pbtxt @@ -0,0 +1,192 @@ +op { + graph_op_name: "RGBToHSV" + endpoint { + name: "image.rgb_to_hsv" + } +} +op { + graph_op_name: "RandomCrop" + visibility: HIDDEN +} +op { + graph_op_name: "RandomGamma" + visibility: HIDDEN +} +op { + graph_op_name: "RandomPoisson" + visibility: HIDDEN +} +op { + graph_op_name: "RandomShuffle" + visibility: HIDDEN +} +op { + graph_op_name: "RandomShuffleQueue" + visibility: HIDDEN +} +op { + graph_op_name: "RandomShuffleQueueV2" + visibility: HIDDEN +} +op { + graph_op_name: "RandomStandardNormal" + visibility: HIDDEN +} +op { + graph_op_name: "RandomUniform" + visibility: HIDDEN +} +op { + graph_op_name: "RandomUniformInt" + visibility: HIDDEN +} +op { + graph_op_name: "Range" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumRecordsProduced" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumRecordsProducedV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumWorkUnitsCompleted" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderNumWorkUnitsCompletedV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderRead" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReadUpTo" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReadUpToV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReadV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderReset" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderResetV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderRestoreState" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderRestoreStateV2" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderSerializeState" + visibility: HIDDEN +} +op { + graph_op_name: "ReaderSerializeStateV2" + visibility: HIDDEN +} +op { + graph_op_name: "RealDiv" + visibility: HIDDEN +} +op { + graph_op_name: "ReciprocalGrad" + visibility: HIDDEN +} +op { + graph_op_name: "RefExit" + visibility: HIDDEN +} +op { + graph_op_name: "RefIdentity" + visibility: HIDDEN +} +op { + graph_op_name: "RefMerge" + visibility: HIDDEN +} +op { + graph_op_name: "Relu" + endpoint { + name: "nn.relu" + } +} +op { + graph_op_name: "Relu6" + visibility: HIDDEN +} +op { + graph_op_name: "Relu6Grad" + visibility: HIDDEN +} +op { + graph_op_name: "ReluGrad" + visibility: HIDDEN +} +op { + graph_op_name: "ResizeArea" + endpoint { + name: "image.resize_area" + } +} +op { + graph_op_name: "ResizeBicubic" + endpoint { + name: "image.resize_bicubic" + } +} +op { + graph_op_name: "ResizeBicubicGrad" + visibility: HIDDEN +} +op { + graph_op_name: "ResizeBilinear" + endpoint { + name: "image.resize_bilinear" + } +} +op { + graph_op_name: "ResizeBilinearGrad" + visibility: HIDDEN +} +op { + graph_op_name: "ResizeNearestNeighbor" + endpoint { + name: "image.resize_nearest_neighbor" + } +} +op { + graph_op_name: "ResizeNearestNeighborGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Restore" + visibility: HIDDEN +} +op { + graph_op_name: "RestoreSlice" + visibility: HIDDEN +} +op { + graph_op_name: "Reverse" + visibility: HIDDEN +} +op { + graph_op_name: "RsqrtGrad" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_S.pbtxt b/tensorflow/core/api_def/python_api/api_def_S.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..0c34730200c88a0c75cd4cb11f3cf8f177583417 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_S.pbtxt @@ -0,0 +1,252 @@ +op { + graph_op_name: "SampleDistortedBoundingBox" + visibility: HIDDEN +} +op { + graph_op_name: "SampleDistortedBoundingBoxV2" + visibility: HIDDEN +} +op { + graph_op_name: "Save" + visibility: HIDDEN +} +op { + graph_op_name: "SaveSlices" + visibility: HIDDEN +} +op { + graph_op_name: "ScalarSummary" + visibility: HIDDEN +} +op { + graph_op_name: "SdcaFprint" + endpoint { + name: "train.sdca_fprint" + } +} +op { + graph_op_name: "SdcaOptimizer" + endpoint { + name: "train.sdca_optimizer" + } +} +op { + graph_op_name: "SdcaShrinkL1" + endpoint { + name: "train.sdca_shrink_l1" + } +} +op { + graph_op_name: "Select" + visibility: HIDDEN +} +op { + graph_op_name: "SelfAdjointEig" + visibility: HIDDEN +} +op { + graph_op_name: "SelfAdjointEigV2" + visibility: HIDDEN +} +op { + graph_op_name: "Selu" + endpoint { + name: "nn.selu" + } +} +op { + graph_op_name: "SeluGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SerializeManySparse" + visibility: HIDDEN +} +op { + graph_op_name: "SerializeSparse" + visibility: HIDDEN +} +op { + graph_op_name: "ShardedFilename" + visibility: HIDDEN +} +op { + graph_op_name: "ShardedFilespec" + visibility: HIDDEN +} +op { + graph_op_name: "Sigmoid" + visibility: HIDDEN +} +op { + graph_op_name: "SigmoidGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Skipgram" + visibility: HIDDEN +} +op { + graph_op_name: "Slice" + visibility: HIDDEN +} +op { + graph_op_name: "Softmax" + visibility: HIDDEN +} +op { + graph_op_name: "SoftmaxCrossEntropyWithLogits" + visibility: HIDDEN +} +op { + graph_op_name: "Softplus" + endpoint { + name: "nn.softplus" + } +} +op { + graph_op_name: "SoftplusGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Softsign" + endpoint { + name: "nn.softsign" + } +} +op { + graph_op_name: "SoftsignGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SpaceToBatch" + visibility: HIDDEN +} +op { + graph_op_name: "SparseAdd" + visibility: HIDDEN +} +op { + graph_op_name: "SparseAddGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SparseConcat" + visibility: HIDDEN +} +op { + graph_op_name: "SparseCross" + visibility: HIDDEN +} +op { + graph_op_name: "SparseFillEmptyRows" + visibility: HIDDEN +} +op { + graph_op_name: "SparseFillEmptyRowsGrad" + visibility: HIDDEN +} +op { + graph_op_name: "SparseMatMul" + visibility: HIDDEN +} +op { + graph_op_name: "SparseReorder" + visibility: HIDDEN +} +op { + graph_op_name: "SparseReshape" + visibility: HIDDEN +} +op { + graph_op_name: "SparseSoftmaxCrossEntropyWithLogits" + visibility: HIDDEN +} +op { + graph_op_name: "SparseSplit" + visibility: HIDDEN +} +op { + graph_op_name: "SparseTensorDenseAdd" + visibility: HIDDEN +} +op { + graph_op_name: "SparseTensorDenseMatMul" + visibility: HIDDEN +} +op { + graph_op_name: "SparseToDense" + visibility: HIDDEN +} +op { + graph_op_name: "Split" + visibility: HIDDEN +} +op { + graph_op_name: "SplitV" + visibility: HIDDEN +} +op { + graph_op_name: "SqrtGrad" + visibility: HIDDEN +} +op { + graph_op_name: "Squeeze" + visibility: HIDDEN +} +op { + graph_op_name: "Stack" + visibility: HIDDEN +} +op { + graph_op_name: "StackClose" + visibility: HIDDEN +} +op { + graph_op_name: "StackCloseV2" + visibility: HIDDEN +} +op { + graph_op_name: "StackPop" + visibility: HIDDEN +} +op { + graph_op_name: "StackPopV2" + visibility: HIDDEN +} +op { + graph_op_name: "StackPush" + visibility: HIDDEN +} +op { + graph_op_name: "StackPushV2" + visibility: HIDDEN +} +op { + graph_op_name: "StackV2" + visibility: HIDDEN +} +op { + graph_op_name: "StringSplit" + visibility: HIDDEN +} +op { + graph_op_name: "Sub" + visibility: HIDDEN +} +op { + graph_op_name: "Sum" + visibility: HIDDEN +} +op { + graph_op_name: "Svd" + visibility: HIDDEN +} +op { + graph_op_name: "Switch" + visibility: HIDDEN +} +op { + graph_op_name: "SymbolicGradient" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_T.pbtxt b/tensorflow/core/api_def/python_api/api_def_T.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..8011a11243f307c4046aba376b39e34c53cd479a --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_T.pbtxt @@ -0,0 +1,196 @@ +op { + graph_op_name: "TFRecordReader" + visibility: HIDDEN +} +op { + graph_op_name: "TFRecordReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "TakeManySparseFromTensorsMap" + visibility: HIDDEN +} +op { + graph_op_name: "Tanh" + visibility: HIDDEN +} +op { + graph_op_name: "TanhGrad" + visibility: HIDDEN +} +op { + graph_op_name: "TemporaryVariable" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArray" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayClose" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayCloseV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayCloseV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayConcat" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayConcatV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayConcatV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGather" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGatherV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGatherV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGrad" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGradV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayGradV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayPack" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayRead" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayReadV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayReadV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayScatter" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayScatterV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayScatterV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySize" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySizeV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySizeV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySplit" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySplitV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArraySplitV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayUnpack" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayWrite" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayWriteV2" + visibility: HIDDEN +} +op { + graph_op_name: "TensorArrayWriteV3" + visibility: HIDDEN +} +op { + graph_op_name: "TensorSummary" + visibility: HIDDEN +} +op { + graph_op_name: "TensorSummaryV2" + visibility: HIDDEN +} +op { + graph_op_name: "TextLineReader" + visibility: HIDDEN +} +op { + graph_op_name: "TextLineReaderV2" + visibility: HIDDEN +} +op { + graph_op_name: "ThreadUnsafeUnigramCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "TileGrad" + visibility: HIDDEN +} +op { + graph_op_name: "TopK" + visibility: HIDDEN +} +op { + graph_op_name: "TopKV2" + visibility: HIDDEN +} +op { + graph_op_name: "TruncateDiv" + visibility: HIDDEN +} +op { + graph_op_name: "TruncateMod" + visibility: HIDDEN +} +op { + graph_op_name: "TruncatedNormal" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_U.pbtxt b/tensorflow/core/api_def/python_api/api_def_U.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..d7c261c63c8dcc259b4d5e77f114dba746538f61 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_U.pbtxt @@ -0,0 +1,8 @@ +op { + graph_op_name: "UniformCandidateSampler" + visibility: HIDDEN +} +op { + graph_op_name: "Unpack" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_V.pbtxt b/tensorflow/core/api_def/python_api/api_def_V.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..18be21a8866580e0a156c5edd034685062baa550 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_V.pbtxt @@ -0,0 +1,8 @@ +op { + graph_op_name: "Variable" + visibility: HIDDEN +} +op { + graph_op_name: "VariableV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_W.pbtxt b/tensorflow/core/api_def/python_api/api_def_W.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..cd8861a98f6e58cd382c56ebc9669cfe34af8688 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_W.pbtxt @@ -0,0 +1,8 @@ +op { + graph_op_name: "WholeFileReader" + visibility: HIDDEN +} +op { + graph_op_name: "WholeFileReaderV2" + visibility: HIDDEN +} diff --git a/tensorflow/core/api_def/python_api/api_def_Z.pbtxt b/tensorflow/core/api_def/python_api/api_def_Z.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..5857b7cf3888718120ccea651cd81e4086afcbc6 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_Z.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "ZerosLike" + visibility: HIDDEN +} diff --git a/tensorflow/core/common_runtime/bfc_allocator.h b/tensorflow/core/common_runtime/bfc_allocator.h index 326e0ffe40cddf612899ad047a6c9b8fea6cad63..20fa05f0d228c754ca0093ca7f360592cdaa23f2 100644 --- a/tensorflow/core/common_runtime/bfc_allocator.h +++ b/tensorflow/core/common_runtime/bfc_allocator.h @@ -362,6 +362,17 @@ class BFCAllocator : public VisitableAllocator { // Structures immutable after construction size_t memory_limit_ = 0; + + inline int Log2FloorNonZeroSlow(uint64 n) { + int r = 0; + while (n > 0) { + r++; + n >>= 1; + } + return r - 1; + } + + // Returns floor(log2(n)). inline int Log2FloorNonZero(uint64 n) { #if defined(__GNUC__) return 63 ^ __builtin_clzll(n); @@ -370,12 +381,7 @@ class BFCAllocator : public VisitableAllocator { _BitScanReverse64(&index, n); return index; #else - int r = 0; - while (n > 0) { - r++; - n >>= 1; - } - return r; + return Log2FloorNonZeroSlow(n); #endif } @@ -425,7 +431,7 @@ class BFCAllocator : public VisitableAllocator { // Stats. AllocatorStats stats_ GUARDED_BY(lock_); - friend class GPUBFCAllocatorBinDebugInfoTest; + friend class GPUBFCAllocatorPrivateMethodsTest; TF_DISALLOW_COPY_AND_ASSIGN(BFCAllocator); }; diff --git a/tensorflow/core/common_runtime/constant_folding.cc b/tensorflow/core/common_runtime/constant_folding.cc index aca68d4c4a84c29eb7bdbe59893a7a6318fbbc37..0398c2a60d1fe4dfeed91e242272f13dd45389b2 100644 --- a/tensorflow/core/common_runtime/constant_folding.cc +++ b/tensorflow/core/common_runtime/constant_folding.cc @@ -460,7 +460,8 @@ Graph* GetConstantGraph( // new constant node. bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device, NodeAndOutput tensor, const Tensor& constant, - const gtl::FlatSet& control_deps) { + const gtl::FlatSet& control_deps, + int64 max_constant_size_in_bytes) { // Be conservative when replacing a tensor with a constant, when not // running on CPU. // 1) If the destination tensor is not an int32 tensor, and has HOST_MEMORY @@ -469,8 +470,9 @@ bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device, // constraint, do not replace it. // 3) If the constant op created does not have a kernel implementation // for the device, do not use it. - // 4) If the size of the constant in bytes is too large (> 10M), do not - // replace it. This prevents the size of the Graph from growing too large. + // 4) If the size of the constant in bytes is too large (> + // max_constant_in_bytes), do not replace it. This prevents the size of the + // Graph from growing too large. // TODO(keveman): Consider adding a new constant op that has a kernel // implementation for all types, but with HostMemory constraint on it's // output. @@ -494,7 +496,7 @@ bool ReplaceTensorWithConstant(Graph* graph, Device* partition_device, return false; } } - if (constant.TotalBytes() > 10 * 1024 * 1024) { + if (constant.TotalBytes() > max_constant_size_in_bytes) { return false; } @@ -613,9 +615,9 @@ Status ConstantFold(const ConstantFoldingOptions& opts, for (size_t c = 0; c < outputs.size(); ++c) { const gtl::FlatSet& control_deps = constant_control_deps[tensors_to_replace[c].first]; - if (ReplaceTensorWithConstant(graph, partition_device, - tensors_to_replace[c], outputs[c], - control_deps)) { + if (ReplaceTensorWithConstant( + graph, partition_device, tensors_to_replace[c], outputs[c], + control_deps, opts.max_constant_size_in_bytes)) { ++num_nodes_replaced; } } diff --git a/tensorflow/core/common_runtime/constant_folding.h b/tensorflow/core/common_runtime/constant_folding.h index e7b1571a81b65cce93ae0a70ad4bd52ce9f31687..e4d724c58a25347db3e40a0d024acf1ac97ea575 100644 --- a/tensorflow/core/common_runtime/constant_folding.h +++ b/tensorflow/core/common_runtime/constant_folding.h @@ -34,6 +34,9 @@ struct ConstantFoldingOptions { // outputs. const std::unordered_map>* shape_map = nullptr; // not owned + // The maximum size of each constant created during constant folding + // optimization. + int64 max_constant_size_in_bytes = 10 * 1024 * 1024; }; // Perform constant folding optimization on "graph". diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc index 2c7c20817ac33bb5f8a3c1ce4e3505f9e347782a..923a4d924936386ce0e06c6355c2a4d0af5cc4a4 100644 --- a/tensorflow/core/common_runtime/constant_folding_test.cc +++ b/tensorflow/core/common_runtime/constant_folding_test.cc @@ -259,6 +259,13 @@ TEST_F(ConstantFoldingTest, TestNoReplaceLargeConstant) { TF_EXPECT_OK(ConstantFold(ConstantFoldingOptions{}, nullptr, Env::Default(), nullptr, &g, &was_mutated)); EXPECT_FALSE(was_mutated); + + // Increase the limit and the concat should now be constant folded. + ConstantFoldingOptions opt; + opt.max_constant_size_in_bytes = 10 * 1024 * 1024 + 4; + TF_EXPECT_OK( + ConstantFold(opt, nullptr, Env::Default(), nullptr, &g, &was_mutated)); + EXPECT_TRUE(was_mutated); } TEST_F(ConstantFoldingTest, TestNoReplaceFunctionCall) { diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc index d886a0230573fc52cc15274ea45252a8629fac96..10356fc7890d1d0b8ce257bea28dbd6d9ddb6835 100644 --- a/tensorflow/core/common_runtime/function.cc +++ b/tensorflow/core/common_runtime/function.cc @@ -569,10 +569,8 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, string target_device = parent_->GetDeviceName(handle); string source_device = opts.source_device; Rendezvous* rendezvous = opts.rendezvous; - // TODO(rohanj): Handle alloc_attrs in Rendezvous::Args. - Rendezvous::Args rendez_args; - Status s = - parent_->GetDeviceContext(target_device, &rendez_args.device_context); + DeviceContext* device_context; + Status s = parent_->GetDeviceContext(target_device, &device_context); if (!s.ok()) { delete frame; delete exec_args; @@ -596,12 +594,14 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, std::vector* remote_args = new std::vector; ProcessFunctionLibraryRuntime::ReceiveTensorsAsync( source_device, target_device, "arg_", src_incarnation, args.size(), - rendez_args, rendezvous, remote_args, + device_context, {}, rendezvous, remote_args, [frame, remote_args, item, source_device, target_device, - target_incarnation, rendezvous, rendez_args, rets, done, + target_incarnation, rendezvous, device_context, rets, done, exec_args](const Status& status) { Status s = status; - s = frame->SetArgs(*remote_args); + if (s.ok()) { + s = frame->SetArgs(*remote_args); + } if (!s.ok()) { delete frame; delete remote_args; @@ -611,7 +611,7 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, } item->exec->RunAsync( *exec_args, [item, frame, rets, done, source_device, target_device, - target_incarnation, rendezvous, rendez_args, + target_incarnation, rendezvous, device_context, remote_args, exec_args](const Status& status) { item->Unref(); Status s = status; @@ -627,7 +627,7 @@ void FunctionLibraryRuntimeImpl::RunRemote(const Options& opts, Handle handle, } s = ProcessFunctionLibraryRuntime::SendTensors( target_device, source_device, "ret_", target_incarnation, - *rets, rendez_args, rendezvous); + *rets, device_context, {}, rendezvous); delete remote_args; delete exec_args; done(s); @@ -643,8 +643,18 @@ void FunctionLibraryRuntimeImpl::Run(const Options& opts, Handle handle, done(errors::Cancelled("")); return; } + Options run_opts = opts; + if (opts.create_rendezvous) { + Rendezvous* rendezvous = new IntraProcessRendezvous(device_mgr_); + run_opts.rendezvous = rendezvous; + run_opts.create_rendezvous = false; + done = [done, rendezvous](const Status& status) { + rendezvous->Unref(); + done(status); + }; + } if (!parent_->IsInstantiatedOnDevice(device_name_, handle)) { - parent_->Run(opts, handle, args, rets, done); + parent_->Run(run_opts, handle, args, rets, done); return; } const FunctionBody* fbody = GetFunctionBody(handle); @@ -658,20 +668,20 @@ void FunctionLibraryRuntimeImpl::Run(const Options& opts, Handle handle, done(s); return; } - DCHECK(opts.runner != nullptr); + DCHECK(run_opts.runner != nullptr); Executor::Args* exec_args = new Executor::Args; // Inherit the step_id from the caller. - exec_args->step_id = opts.step_id; - exec_args->rendezvous = opts.rendezvous; - exec_args->stats_collector = opts.stats_collector; + exec_args->step_id = run_opts.step_id; + exec_args->rendezvous = run_opts.rendezvous; + exec_args->stats_collector = run_opts.stats_collector; exec_args->call_frame = frame; - exec_args->cancellation_manager = opts.cancellation_manager; - exec_args->step_container = opts.step_container; - exec_args->runner = *opts.runner; + exec_args->cancellation_manager = run_opts.cancellation_manager; + exec_args->step_container = run_opts.step_container; + exec_args->runner = *run_opts.runner; - if (opts.remote_execution) { - RunRemote(opts, handle, args, rets, exec_args, item, done); + if (run_opts.remote_execution) { + RunRemote(run_opts, handle, args, rets, exec_args, item, done); return; } diff --git a/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc b/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc index b7554e5b82c642c1833168d6b17fbd0179c960c5..00ef130d34bbbe06ad9dabae124ff3fa0d38450a 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_bfc_allocator_test.cc @@ -354,12 +354,13 @@ BENCHMARK(BM_AllocationDelayed)->Arg(1)->Arg(10)->Arg(100)->Arg(1000); } // namespace -class GPUBFCAllocatorBinDebugInfoTest : public ::testing::Test { +class GPUBFCAllocatorPrivateMethodsTest : public ::testing::Test { protected: - // This test method is called from a test. The reason for this is that this - // class is a friend class to BFCAllocator, but tests are not, so only this - // method can access the type BFCAllocator::BinDebugInfo. - void testBinDebugInfo() { + // The following test methods are called from tests. The reason for this is + // that this class is a friend class to BFCAllocator, but tests are not, so + // only methods inside this class can access private members of BFCAllocator. + + void TestBinDebugInfo() { GPUBFCAllocator a(0, 1 << 30); std::vector initial_ptrs; @@ -436,9 +437,24 @@ class GPUBFCAllocatorBinDebugInfoTest : public ::testing::Test { } } } + + void TestLog2FloorNonZeroSlow() { + GPUBFCAllocator a(0 /* device_id */, 1 /* total_memory */); + EXPECT_EQ(-1, a.Log2FloorNonZeroSlow(0)); + EXPECT_EQ(0, a.Log2FloorNonZeroSlow(1)); + EXPECT_EQ(1, a.Log2FloorNonZeroSlow(2)); + EXPECT_EQ(1, a.Log2FloorNonZeroSlow(3)); + EXPECT_EQ(9, a.Log2FloorNonZeroSlow(1023)); + EXPECT_EQ(10, a.Log2FloorNonZeroSlow(1024)); + EXPECT_EQ(10, a.Log2FloorNonZeroSlow(1025)); + } }; -TEST_F(GPUBFCAllocatorBinDebugInfoTest, BinDebugInfo) { testBinDebugInfo(); } +TEST_F(GPUBFCAllocatorPrivateMethodsTest, BinDebugInfo) { TestBinDebugInfo(); } + +TEST_F(GPUBFCAllocatorPrivateMethodsTest, Log2FloorNonZeroSlow) { + TestLog2FloorNonZeroSlow(); +} } // namespace tensorflow diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc index 3324e833ff71517f17b7d0ad54ee22a11c81d0e3..12d44cc6b7d0b724b5fe1c427b31e455eeca07fe 100644 --- a/tensorflow/core/common_runtime/gpu/gpu_device.cc +++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc @@ -475,7 +475,8 @@ void BaseGPUDevice::ComputeAsync(AsyncOpKernel* op_kernel, // When TraceMe profiling is off (which is the default), the // following TraceMe constructor is simply a conditional test of // false value. Measurements show that its overhead is negligible. - port::Tracing::TraceMe activity(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe activity(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); gpu::cuda::ScopedActivateExecutorContext scoped_activation{stream->parent()}; op_kernel->ComputeAsync(context, done); } diff --git a/tensorflow/core/common_runtime/graph_optimizer.cc b/tensorflow/core/common_runtime/graph_optimizer.cc index ff99db95326ebe7ec51e4ca7190b667624314e8d..def185e52280bf004bf67cb1daef675c2f6ccff5 100644 --- a/tensorflow/core/common_runtime/graph_optimizer.cc +++ b/tensorflow/core/common_runtime/graph_optimizer.cc @@ -61,6 +61,10 @@ void GraphOptimizer::Optimize( if (opts_.do_constant_folding()) { ConstantFoldingOptions cf_opts; cf_opts.shape_map = shape_map; + if (opts_.max_folded_constant_in_bytes() > 0) { + cf_opts.max_constant_size_in_bytes = + opts_.max_folded_constant_in_bytes(); + } bool was_mutated; ConstantFold(cf_opts, runtime, env, device, g, &was_mutated) .IgnoreError(); diff --git a/tensorflow/core/common_runtime/graph_runner.cc b/tensorflow/core/common_runtime/graph_runner.cc index d0f9e6ed18fcb583937cec22f44faab9e5cf66ef..a21304f7ef843706d564bd3f3a511324fd3189d6 100644 --- a/tensorflow/core/common_runtime/graph_runner.cc +++ b/tensorflow/core/common_runtime/graph_runner.cc @@ -109,6 +109,17 @@ Status GraphRunner::Run(Graph* graph, FunctionLibraryRuntime* function_library, return errors::NotFound("Cannot find a device for GraphRunner."); } + if (function_library && function_library->device() && + function_library->device()->device_type() != cpu_device_->device_type()) { + // We are running on a CPU but the function library is for a non-CPU device, + // so just ignore the function_library. + // TODO(matthewmurray) Can we create a new FunctionLibraryRuntime that is + // identical to function_library except that it uses CPU? + VLOG(1) << "Cannot run on CPU device with a function library for a " + << function_library->device()->device_type() << " device."; + function_library = nullptr; + } + // TODO(vrv): Instead of copying the entire graph, consider modifying // the existing graph, and then removing those removed edges. // prior to returning. diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h index 5951b3b6a17950eecd59df2d20573c1896896a04..53e80b1ee302761c04df1ec9d242d9edd2a1f510 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h @@ -51,7 +51,7 @@ class MklCPUAllocator : public Allocator { // Constructor and other standard functions /// Environment variable that user can set to upper bound on memory allocation - static constexpr const char kMaxLimitStr[] = "TF_MKL_ALLOC_MAX_BYTES"; + static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES"; /// Default upper limit on allocator size - 64GB static const size_t kDefaultMaxLimit = 64LL << 30; @@ -146,7 +146,7 @@ class MklCPUAllocator : public Allocator { static const bool kAllowGrowth = true; /// Name - static constexpr const char kName[] = "mklcpu"; + static constexpr const char* kName = "mklcpu"; /// The alignment that we need for the allocations static const size_t kAlignment = 64; diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc index cfefaa92e4dd023026e0ca2b51821c0ca4e5c8ae..a67411cd2e23e0d1e9090c7a40c6f715d71d7c85 100644 --- a/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc +++ b/tensorflow/core/common_runtime/mkl_cpu_allocator_test.cc @@ -23,8 +23,6 @@ limitations under the License. namespace tensorflow { -constexpr char MklCPUAllocator::kMaxLimitStr[]; - TEST(MKLBFCAllocatorTest, TestMaxLimit) { AllocatorStats stats; setenv(MklCPUAllocator::kMaxLimitStr, "1000", 1); diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index 68ff28e4d8cceff9b97ace93d509dcd82391eb8d..c4114ff8739f15f0993f9164e2046c94a3c586bc 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -95,7 +95,8 @@ string ProcessFunctionLibraryRuntime::ObtainFunctionTarget( Status ProcessFunctionLibraryRuntime::SendTensors( const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, - gtl::ArraySlice tensors_to_send, const Rendezvous::Args& args, + gtl::ArraySlice tensors_to_send, DeviceContext* device_context, + const std::vector& alloc_attrs, Rendezvous* rendezvous) { std::vector keys; for (int i = 0; i < tensors_to_send.size(); ++i) { @@ -104,8 +105,8 @@ Status ProcessFunctionLibraryRuntime::SendTensors( target_device, name, FrameAndIter(0, 0)); keys.push_back(key); } - TF_RETURN_IF_ERROR( - SendTensorsToRendezvous(rendezvous, args, keys, tensors_to_send)); + TF_RETURN_IF_ERROR(SendTensorsToRendezvous( + rendezvous, device_context, alloc_attrs, keys, tensors_to_send)); return Status::OK(); } @@ -113,7 +114,8 @@ Status ProcessFunctionLibraryRuntime::SendTensors( void ProcessFunctionLibraryRuntime::ReceiveTensorsAsync( const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, int64 num_tensors, - const Rendezvous::Args& args, Rendezvous* rendezvous, + DeviceContext* device_context, + const std::vector& alloc_attrs, Rendezvous* rendezvous, std::vector* received_tensors, const StatusCallback& done) { std::vector keys; for (int64 i = 0; i < num_tensors; ++i) { @@ -123,7 +125,7 @@ void ProcessFunctionLibraryRuntime::ReceiveTensorsAsync( keys.push_back(key); } RecvOutputsFromRendezvousAsync( - rendezvous, args, keys, received_tensors, + rendezvous, device_context, alloc_attrs, keys, received_tensors, [done](const Status& status) { done(status); }); } @@ -265,8 +267,8 @@ void ProcessFunctionLibraryRuntime::Run( if (flr != nullptr) { auto rendezvous = opts.rendezvous; string source_device = opts.source_device; - Rendezvous::Args rendez_args; - Status s = GetDeviceContext(source_device, &rendez_args.device_context); + DeviceContext* device_context; + Status s = GetDeviceContext(source_device, &device_context); if (!s.ok()) { done(s); return; @@ -281,15 +283,18 @@ void ProcessFunctionLibraryRuntime::Run( // Send the args over to the target device. s = SendTensors(source_device, target_device, "arg_", src_incarnation, args, - rendez_args, rendezvous); + device_context, opts.args_alloc_attrs, rendezvous); if (!s.ok()) { done(s); return; } + const std::vector& rets_alloc_attrs = + opts.rets_alloc_attrs; std::vector* remote_rets = new std::vector; flr->Run(opts, handle, args, remote_rets, [source_device, target_device, target_incarnation, rendezvous, - remote_rets, rets, done, rendez_args](const Status& status) { + device_context, rets_alloc_attrs, remote_rets, rets, + done](const Status& status) { if (!status.ok()) { delete remote_rets; done(status); @@ -299,8 +304,9 @@ void ProcessFunctionLibraryRuntime::Run( delete remote_rets; // Now receive the return values from the target. ReceiveTensorsAsync(target_device, source_device, "ret_", - target_incarnation, num_returns, rendez_args, - rendezvous, rets, done); + target_incarnation, num_returns, + device_context, rets_alloc_attrs, rendezvous, + rets, done); }); return; } diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.h b/tensorflow/core/common_runtime/process_function_library_runtime.h index 9f03de0f76a429b48cbc337173187f5ed1d692a7..85717739d0c61006995f1961b3285c53ee0ef57f 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.h +++ b/tensorflow/core/common_runtime/process_function_library_runtime.h @@ -60,26 +60,33 @@ class ProcessFunctionLibraryRuntime { // Sends `tensors_to_send` from `source_device` to `target_device` using // `rendezvous`. `key_prefix` is used as a prefix for the keys sent to the - // Rendezvous. Method takes references on each of the `tensors_to_send`. - // Method doesn't block. + // Rendezvous. `device_context` should be the DeviceContext of the device + // doing the sending. `alloc_attrs` should either be empty or be the size of + // `tensors_to_send` and indicates how the input tensors are allocated. Method + // takes references on each of the `tensors_to_send`. Method doesn't block. static Status SendTensors(const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, gtl::ArraySlice tensors_to_send, - const Rendezvous::Args& args, + DeviceContext* device_context, + const std::vector& alloc_attrs, Rendezvous* rendezvous); typedef std::function StatusCallback; // Receives `received_tensors` from `target_device` (originally sent from // `source_device`) using `rendezvous`. Uses `key_prefix` to construct the - // keys to be retrieved. Method doesn't block and calls `done` when - // `num_tensors` are fetched. + // keys to be retrieved. `device_context` should be for the device receiving + // the tensors. `alloc_attrs` indicates how to allocate the received + // tensors and should either be empty or `num_tensors` in size. Method doesn't + // block and calls `done` when `num_tensors` are fetched. static void ReceiveTensorsAsync( const string& source_device, const string& target_device, const string& key_prefix, int64 src_incarnation, int64 num_tensors, - const Rendezvous::Args& args, Rendezvous* rendezvous, - std::vector* received_tensors, const StatusCallback& done); + DeviceContext* device_context, + const std::vector& alloc_attrs, + Rendezvous* rendezvous, std::vector* received_tensors, + const StatusCallback& done); static const char kDefaultFLRDevice[]; // Returns the FunctionLibraryRuntime for the corresponding device_name. diff --git a/tensorflow/core/common_runtime/renamed_device.h b/tensorflow/core/common_runtime/renamed_device.h index 0158e18cedc3b9b136258085641492c94de9e612..22a70fbdfaea3d77440e777ac5261af8c3aeb551 100644 --- a/tensorflow/core/common_runtime/renamed_device.h +++ b/tensorflow/core/common_runtime/renamed_device.h @@ -37,6 +37,13 @@ class RenamedDevice : public Device { return underlying_->RequiresRecordingAccessedTensors(); } + const DeviceBase* UnderlyingDevice() const override { + return underlying_->UnderlyingDevice(); + } + DeviceBase* UnderlyingDevice() override { + return underlying_->UnderlyingDevice(); + } + const CpuWorkerThreads* tensorflow_cpu_worker_threads() const override { return underlying_->tensorflow_cpu_worker_threads(); } diff --git a/tensorflow/core/common_runtime/rendezvous_util.cc b/tensorflow/core/common_runtime/rendezvous_util.cc index a0d409e7735d27d17af6e9d76299a3ee7beb944b..a1e31016c2bc93aeae76175320255e0d43602265 100644 --- a/tensorflow/core/common_runtime/rendezvous_util.cc +++ b/tensorflow/core/common_runtime/rendezvous_util.cc @@ -16,35 +16,55 @@ limitations under the License. namespace tensorflow { -Status SendTensorsToRendezvous(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - gtl::ArraySlice tensors_to_send) { +Status SendTensorsToRendezvous( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, gtl::ArraySlice tensors_to_send) { if (keys.size() != tensors_to_send.size()) { return errors::InvalidArgument( "keys and tensors_to_send are not the same size. keys.size() = ", keys.size(), "; tensors_to_send.size() = ", tensors_to_send.size()); } + if (!alloc_attrs.empty() && (keys.size() != alloc_attrs.size())) { + return errors::InvalidArgument( + "keys and alloc_attrs are not the same size. ", + "keys.size() = ", keys.size(), + "; alloc_attrs.size() = ", alloc_attrs.size()); + } + Rendezvous::ParsedKey parsed; for (int i = 0; i < keys.size(); ++i) { + Rendezvous::Args rendez_args; + rendez_args.device_context = device_context; + if (!alloc_attrs.empty()) { + rendez_args.alloc_attrs = alloc_attrs[i]; + } TF_RETURN_IF_ERROR(Rendezvous::ParseKey(keys[i], &parsed)); TF_RETURN_IF_ERROR( - rendezvous->Send(parsed, args, tensors_to_send[i], false)); + rendezvous->Send(parsed, rendez_args, tensors_to_send[i], false)); } return Status::OK(); } -void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - std::vector* received_tensors, - const StatusCallback& done) { +void RecvOutputsFromRendezvousAsync( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, std::vector* received_tensors, + const StatusCallback& done) { if (keys.empty()) { done(Status::OK()); return; } + if (!alloc_attrs.empty() && (keys.size() != alloc_attrs.size())) { + done(errors::InvalidArgument( + "keys and alloc_attrs are not the same size. ", "keys.size() = ", + keys.size(), "; alloc_attrs.size() = ", alloc_attrs.size())); + } + received_tensors->reserve(keys.size()); - std::vector> arguments; + std::vector< + std::tuple> + arguments; for (int i = 0; i < keys.size(); ++i) { Rendezvous::ParsedKey parsed; Status s = Rendezvous::ParseKey(keys[i], &parsed); @@ -53,8 +73,12 @@ void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, done(s); return; } - arguments.push_back( - std::make_tuple(keys[i], &((*received_tensors)[i]), parsed)); + AllocatorAttributes alloc_attr; + if (!alloc_attrs.empty()) { + alloc_attr = alloc_attrs[i]; + } + arguments.emplace_back(keys[i], &((*received_tensors)[i]), parsed, + alloc_attr); } typedef struct { @@ -68,8 +92,12 @@ void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, const string& key = std::get<0>(p); Tensor* val = std::get<1>(p); Rendezvous::ParsedKey parsed = std::get<2>(p); + Rendezvous::Args rendez_args; + rendez_args.device_context = device_context; + rendez_args.alloc_attrs = std::get<3>(p); + rendezvous->RecvAsync( - parsed, args, + parsed, rendez_args, [val, done, key, call_state](const Status& s, const Rendezvous::Args& send_args, const Rendezvous::Args& recv_args, diff --git a/tensorflow/core/common_runtime/rendezvous_util.h b/tensorflow/core/common_runtime/rendezvous_util.h index a54f8c3f94855780951e24319fd37e345400748e..3b6354603b2925dd7a1d2abe34308e9c8865f6bb 100644 --- a/tensorflow/core/common_runtime/rendezvous_util.h +++ b/tensorflow/core/common_runtime/rendezvous_util.h @@ -24,17 +24,25 @@ namespace tensorflow { typedef std::map NamedTensors; typedef std::function StatusCallback; -// Uses `rendezvous` to send tensors in `in`. -Status SendTensorsToRendezvous(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - gtl::ArraySlice tensors_to_send); - -void RecvOutputsFromRendezvousAsync(Rendezvous* rendezvous, - const Rendezvous::Args& args, - const std::vector& keys, - std::vector* received_tensors, - const StatusCallback& done); +// Uses `rendezvous` to send tensors in `tensors_to_send`. `device_context` +// should be the DeviceContext associated with the source of the tensors. +// `alloc_attrs` contains information about how the `tensors_to_send` are +// allocated. `alloc_attrs` should either be {} or should match the length of +// `keys`. +Status SendTensorsToRendezvous( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, gtl::ArraySlice tensors_to_send); + +// Uses `rendezvous` to obtain tensors. `device_context` should be the +// DeviceContext associated with the receiving device. `alloc_attrs` contains +// information as how to store the received tensors. Should be {} or match the +// length of `keys`. +void RecvOutputsFromRendezvousAsync( + Rendezvous* rendezvous, DeviceContext* device_context, + const std::vector& alloc_attrs, + const std::vector& keys, std::vector* received_tensors, + const StatusCallback& done); Status RecvOutputsFromRendezvous(Rendezvous* rendezvous, NamedTensors* out, const Rendezvous::Args& args); diff --git a/tensorflow/core/common_runtime/rendezvous_util_test.cc b/tensorflow/core/common_runtime/rendezvous_util_test.cc index 8ee9f4d52263879230a7ab94b8105c578d2dfa2b..093fa7921f56370b0ccd21576991f9ab82b2aa7f 100644 --- a/tensorflow/core/common_runtime/rendezvous_util_test.cc +++ b/tensorflow/core/common_runtime/rendezvous_util_test.cc @@ -52,15 +52,14 @@ string MakeStringKey(const string& name) { TEST_F(RendezvousUtilTest, SendBeforeRecv) { // Fire off sends before receive the tensors. - Rendezvous::Args args; TF_ASSERT_OK(SendTensorsToRendezvous( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, {V("hello1"), V("hello2")})); Notification n; std::vector received_keys; RecvOutputsFromRendezvousAsync( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, &received_keys, [&n](const Status& status) { n.Notify(); }); n.WaitForNotification(); @@ -71,16 +70,14 @@ TEST_F(RendezvousUtilTest, SendBeforeRecv) { TEST_F(RendezvousUtilTest, RecvBeforeSend) { // Fire off recvs, wait for a notification in the callback. - Rendezvous::Args args; - Notification n; std::vector received_keys; RecvOutputsFromRendezvousAsync( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, &received_keys, [&n](const Status& status) { n.Notify(); }); TF_ASSERT_OK(SendTensorsToRendezvous( - rendez_, args, {MakeStringKey("hello1"), MakeStringKey("hello2")}, + rendez_, nullptr, {}, {MakeStringKey("hello1"), MakeStringKey("hello2")}, {V("hello1"), V("hello2")})); n.WaitForNotification(); diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc index 2a0bdc9a7b40bc2e51c28cce849909cb323783ce..1ed5eb3f228674054ecf9bb11505913f6549e460 100644 --- a/tensorflow/core/common_runtime/shape_refiner.cc +++ b/tensorflow/core/common_runtime/shape_refiner.cc @@ -148,6 +148,7 @@ Status ShapeRefiner::InferShapesForFunction( } ShapeRefiner refiner(graph.versions().producer(), &function_library); + refiner.set_disable_constant_propagation(disable_constant_propagation_); refiner.set_function_library_for_shape_inference(&function_library); if (keep_nested_shapes) refiner.set_keep_nested_shape_inferences(); diff --git a/tensorflow/core/common_runtime/shape_refiner.h b/tensorflow/core/common_runtime/shape_refiner.h index bf4c6d88916cc88f9e3cea30d30b126a660b3781..570b4db1635d52765d7ec509bf2b20d78502160b 100644 --- a/tensorflow/core/common_runtime/shape_refiner.h +++ b/tensorflow/core/common_runtime/shape_refiner.h @@ -164,6 +164,10 @@ class ShapeRefiner { function_library_ = lib; } + bool function_shape_inference_supported() const { + return function_library_ != nullptr; + } + // Call this to keep nested shapes information for user-defined functions: // nested inferences will be available on the ExtendedInferenceContext for // each function node, forming a tree of shape inferences corresponding to the @@ -206,7 +210,7 @@ class ShapeRefiner { // - outer_context will contain output shapes inferred from input shapes // - outer_context will contain nested inferences collection, iff // keep_nested_shapes is true - static Status InferShapesForFunction( + Status InferShapesForFunction( const tensorflow::FunctionLibraryDefinition& function_library, const tensorflow::FunctionDef& function_def, bool keep_nested_shapes, ExtendedInferenceContext* outer_context); diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc index 23ccca1c9459f6cf90408bd42a24660067b332cc..5aa01376ab047e7613ba7403bb32859a83a09f5a 100644 --- a/tensorflow/core/common_runtime/threadpool_device.cc +++ b/tensorflow/core/common_runtime/threadpool_device.cc @@ -48,7 +48,8 @@ void ThreadPoolDevice::Compute(OpKernel* op_kernel, OpKernelContext* context) { // When TraceMe profiling is off (which is the default), the // following TraceMe constructor is simply a conditional test of // false value. Measurements show that its overhead is negligible. - port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string()); + port::Tracing::TraceMe trace_me(op_kernel->name(), op_kernel->type_string(), + op_kernel->IsExpensive()); if (port::Tracing::IsActive()) { // TODO(pbar) We really need a useful identifier of the graph node. const uint64 id = Hash64(op_kernel->name()); diff --git a/tensorflow/core/debug/debug_graph_utils.cc b/tensorflow/core/debug/debug_graph_utils.cc index 2559808b59a82d092b3d9e312402db0aeb1cc508..4539ea5c0cb6a11e783fc14c7957065d975da2c3 100644 --- a/tensorflow/core/debug/debug_graph_utils.cc +++ b/tensorflow/core/debug/debug_graph_utils.cc @@ -221,21 +221,26 @@ Status DebugNodeInserter::InsertNodes( } void DebugNodeInserter::DeparallelizeWhileLoops(Graph* graph, Device* device) { + bool deparallelized_a_loop = false; for (Node* node : graph->nodes()) { if (node->IsEnter()) { const AttrValue* parallel_iterations = node->attrs().Find("parallel_iterations"); if (parallel_iterations && parallel_iterations->i() > 1) { - LOG(INFO) << "For debugging, tfdbg is changing the " - << "parallel_iterations attribute of the Enter/RefEnter " - << "node \"" << node->name() << "\" on device \"" - << device->name() << "\" from " << parallel_iterations->i() - << " to 1. (This does not affect subsequent non-debug " - << "runs.)"; + deparallelized_a_loop = true; + VLOG(1) << "Changing the parallel_iterations attribute of the " + << "Enter/RefEnter node \"" << node->name() << "\" on device \"" + << device->name() << "\" from " << parallel_iterations->i() + << " to 1."; node->AddAttr("parallel_iterations", 1); } } } + if (deparallelized_a_loop) { + LOG(INFO) << "For debugging, tfdbg has set the parallel_iterations " + << "attribute of all scheduled Enter/RefEnter nodes to 1. (This " + << "does not affect subsequent non-debug runs.)"; + } } // static diff --git a/tensorflow/core/distributed_runtime/graph_mgr.cc b/tensorflow/core/distributed_runtime/graph_mgr.cc index 7a93b7406c93b2063dfa7067d5cf9f705d4c3f2b..391ffda25c0944490fdac6749d137b97f45d9139 100644 --- a/tensorflow/core/distributed_runtime/graph_mgr.cc +++ b/tensorflow/core/distributed_runtime/graph_mgr.cc @@ -337,8 +337,8 @@ Status GraphMgr::SendInputs(const int64 step_id, const NamedTensors& in) { keys.push_back(p.first); tensors_to_send.push_back(p.second); } - Status s = SendTensorsToRendezvous(rendezvous, Rendezvous::Args(), keys, - tensors_to_send); + Status s = + SendTensorsToRendezvous(rendezvous, nullptr, {}, keys, tensors_to_send); rendezvous->Unref(); return s; } @@ -362,7 +362,7 @@ void GraphMgr::RecvOutputsAsync(const int64 step_id, NamedTensors* out, received_keys->push_back(p.second); } RecvOutputsFromRendezvousAsync( - rendezvous, Rendezvous::Args(), keys, received_keys, + rendezvous, nullptr, {}, keys, received_keys, [done, rendezvous, received_keys, out, keys](const Status s) { rendezvous->Unref(); for (int i = 0; i < keys.size(); ++i) { @@ -420,8 +420,7 @@ void GraphMgr::ExecuteAsync(const string& handle, const int64 step_id, keys.push_back(p.first); tensors_to_send.push_back(p.second); } - s = SendTensorsToRendezvous(rendezvous, Rendezvous::Args(), keys, - tensors_to_send); + s = SendTensorsToRendezvous(rendezvous, nullptr, {}, keys, tensors_to_send); } if (!s.ok()) { diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index 995422644a6b38c3c51955563c60bc7e8f059a72..f7fce1d0ec5bf3cd06d89b67fc6665874f1b2dff 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -746,18 +746,22 @@ void MasterSession::ReffedClientGraph::ProcessStats(int64 step_id, Status::OK()); } // Assemble all stats for this timeline into a merged StepStats. - StepStats step_stats_proto; if (pss->collect_timeline) { - step_stats_proto = pss->rpc_stats; + StepStats step_stats_proto; + step_stats_proto.Swap(&pss->rpc_stats); for (size_t i = 0; i < partitions_.size(); ++i) { - const StepStats& ss = pss->step_stats[i]; - step_stats_proto.MergeFrom(ss); + step_stats_proto.MergeFrom(pss->step_stats[i]); + pss->step_stats[i].Clear(); } - stats_publisher_->PublishStatsProto(step_stats_proto); + pss->step_stats.clear(); // Copy the stats back, but only for on-demand profiling to avoid slowing // down calls that trigger the automatic profiling. if (options.trace_level() == RunOptions::FULL_TRACE) { resp->mutable_step_stats()->Swap(&step_stats_proto); + } else { + // If FULL_TRACE, it can be fetched from Session API, no need for + // duplicated publishing. + stats_publisher_->PublishStatsProto(step_stats_proto); } } } diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD index a8af124e2bb33f68e885ed09ed79ea1f403736d3..5190288e8835ee81566db3bfa52a115c6d48667f 100644 --- a/tensorflow/core/distributed_runtime/rpc/BUILD +++ b/tensorflow/core/distributed_runtime/rpc/BUILD @@ -182,6 +182,7 @@ cc_library( srcs = ["grpc_worker_service_impl.cc"], hdrs = ["grpc_worker_service_impl.h"], deps = [ + ":grpc_namespace_compat", ":grpc_serialization_traits", "//tensorflow/core:worker_proto_cc", "//tensorflow/core/distributed_runtime:worker_interface", @@ -228,12 +229,22 @@ cc_library( srcs = ["grpc_master_service_impl.cc"], hdrs = ["grpc_master_service_impl.h"], deps = [ + ":grpc_namespace_compat", ":grpc_serialization_traits", "//tensorflow/core:master_proto_cc", "@grpc//:grpc++_unsecure", ], ) +cc_library( + name = "grpc_namespace_compat", + srcs = [], + hdrs = ["grpc_namespace_compat.h"], + deps = [ + "@grpc//:grpc++_unsecure", + ], +) + cc_library( name = "grpc_serialization_traits", srcs = [], diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc index 17d0047eb2c83dd32cfa655fe4560ae33fd938d2..d998d51058c5e3178a015770b40f6f637ccf8088 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc +++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.cc @@ -49,74 +49,74 @@ MasterService::Stub::Stub( const std::shared_ptr< ::grpc::ChannelInterface>& channel) : channel_(channel), rpcmethod_CreateSession_(grpcMasterService_method_names[0], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel), + ::grpc::RpcMethod::NORMAL_RPC, channel), rpcmethod_ExtendSession_(grpcMasterService_method_names[1], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel), + ::grpc::RpcMethod::NORMAL_RPC, channel), rpcmethod_PartialRunSetup_(grpcMasterService_method_names[2], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel), + ::grpc::RpcMethod::NORMAL_RPC, channel), rpcmethod_RunStep_(grpcMasterService_method_names[3], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel), + ::grpc::RpcMethod::NORMAL_RPC, channel), rpcmethod_CloseSession_(grpcMasterService_method_names[4], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel), + ::grpc::RpcMethod::NORMAL_RPC, channel), rpcmethod_ListDevices_(grpcMasterService_method_names[5], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel), + ::grpc::RpcMethod::NORMAL_RPC, channel), rpcmethod_Reset_(grpcMasterService_method_names[6], - ::grpc::internal::RpcMethod::NORMAL_RPC, channel) {} + ::grpc::RpcMethod::NORMAL_RPC, channel) {} ::grpc::Status MasterService::Stub::CreateSession( ::grpc::ClientContext* context, const CreateSessionRequest& request, CreateSessionResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CreateSession_, context, request, response); } ::grpc::Status MasterService::Stub::ExtendSession( ::grpc::ClientContext* context, const ExtendSessionRequest& request, ExtendSessionResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ExtendSession_, context, request, response); } ::grpc::Status MasterService::Stub::PartialRunSetup( ::grpc::ClientContext* context, const PartialRunSetupRequest& request, PartialRunSetupResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_PartialRunSetup_, context, request, response); } ::grpc::Status MasterService::Stub::RunStep(::grpc::ClientContext* context, const RunStepRequest& request, RunStepResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_RunStep_, context, request, response); } ::grpc::Status MasterService::Stub::CloseSession( ::grpc::ClientContext* context, const CloseSessionRequest& request, CloseSessionResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_CloseSession_, context, request, response); } ::grpc::Status MasterService::Stub::ListDevices( ::grpc::ClientContext* context, const ListDevicesRequest& request, ListDevicesResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_ListDevices_, context, request, response); } ::grpc::Status MasterService::Stub::Reset(::grpc::ClientContext* context, const ResetRequest& request, ResetResponse* response) { - return ::grpc::internal::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context, + return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_Reset_, context, request, response); } MasterService::AsyncService::AsyncService() { for (int i = 0; i < 7; ++i) { - AddMethod(new ::grpc::internal::RpcServiceMethod( + AddMethod(new ::grpc::RpcServiceMethod( grpcMasterService_method_names[i], - ::grpc::internal::RpcMethod::NORMAL_RPC, + ::grpc::RpcMethod::NORMAL_RPC, nullptr)); ::grpc::Service::MarkMethodAsync(i); } diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h index 412395c52635d5c3cda95dddea50f7cd2d8c8e4f..131de2863f95e86d519c381ef8e100a80fa6561a 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h +++ b/tensorflow/core/distributed_runtime/rpc/grpc_master_service_impl.h @@ -25,6 +25,7 @@ limitations under the License. #include "grpc++/impl/codegen/stub_options.h" #include "grpc++/impl/codegen/sync_stream.h" +#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h" #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h" #include "tensorflow/core/protobuf/master.pb.h" @@ -107,13 +108,13 @@ class MasterService final { private: std::shared_ptr< ::grpc::ChannelInterface> channel_; - const ::grpc::internal::RpcMethod rpcmethod_CreateSession_; - const ::grpc::internal::RpcMethod rpcmethod_ExtendSession_; - const ::grpc::internal::RpcMethod rpcmethod_PartialRunSetup_; - const ::grpc::internal::RpcMethod rpcmethod_RunStep_; - const ::grpc::internal::RpcMethod rpcmethod_CloseSession_; - const ::grpc::internal::RpcMethod rpcmethod_ListDevices_; - const ::grpc::internal::RpcMethod rpcmethod_Reset_; + const ::grpc::RpcMethod rpcmethod_CreateSession_; + const ::grpc::RpcMethod rpcmethod_ExtendSession_; + const ::grpc::RpcMethod rpcmethod_PartialRunSetup_; + const ::grpc::RpcMethod rpcmethod_RunStep_; + const ::grpc::RpcMethod rpcmethod_CloseSession_; + const ::grpc::RpcMethod rpcmethod_ListDevices_; + const ::grpc::RpcMethod rpcmethod_Reset_; }; static std::unique_ptr NewStub( const std::shared_ptr< ::grpc::ChannelInterface>& channel, diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h b/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h new file mode 100644 index 0000000000000000000000000000000000000000..c178927f5d5411e30bee2470b8b544ff76c28396 --- /dev/null +++ b/tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h @@ -0,0 +1,32 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_ +#define THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_ + +// This file is a transitional place-holder until gRPC versions consistently +// use namespace grpc::internal for library-internal structures + +namespace grpc { +// ensure internal namespace exists +namespace internal { +// bring in contents of external namespace +using namespace ::grpc; +} // namespace internal +// bring in contents of internal namespace +using namespace internal; +} // namespace grpc + +#endif // THIRD_PARTY_TENSORFLOW_CORE_DISTRIBUTED_RUNTIME_RPC_GRPC_NAMESPACE_COMPAT_H_ diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc index 348c6dc98bd5bf8a4e6c0a1def8593a858fe6062..80a2f89337c6914dd871c4df346016d70d0f4093 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc +++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.cc @@ -58,9 +58,9 @@ namespace grpc { WorkerService::AsyncService::AsyncService() { for (int i = 0; i < kGrpcNumWorkerMethods; ++i) { - AddMethod(new ::grpc::internal::RpcServiceMethod( + AddMethod(new ::grpc::RpcServiceMethod( GrpcWorkerMethodName(static_cast(i)), - ::grpc::internal::RpcMethod::NORMAL_RPC, nullptr)); + ::grpc::RpcMethod::NORMAL_RPC, nullptr)); ::grpc::Service::MarkMethodAsync(i); } } diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h index e9862a61a3f4ece2218b281d9a78b8ff4d59594f..c8a8b5778e8ad98f9237d0b7f4f04f19beb1ac11 100644 --- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h +++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h @@ -26,6 +26,7 @@ limitations under the License. #include "grpc++/impl/codegen/sync_stream.h" #include "grpc++/support/byte_buffer.h" +#include "tensorflow/core/distributed_runtime/rpc/grpc_namespace_compat.h" #include "tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h" #include "tensorflow/core/distributed_runtime/tensor_coding.h" #include "tensorflow/core/protobuf/worker.pb.h" diff --git a/tensorflow/core/distributed_runtime/scheduler.cc b/tensorflow/core/distributed_runtime/scheduler.cc index 844a0643e62f63719027ad3c922a26a0f7b92505..4766f4c33b654481f7d99ab82939e33e77564771 100644 --- a/tensorflow/core/distributed_runtime/scheduler.cc +++ b/tensorflow/core/distributed_runtime/scheduler.cc @@ -226,7 +226,6 @@ Microseconds GreedyScheduler::ComputeSchedule( while (!event_queue.empty()) { Event event = event_queue.top(); event_queue.pop(); - Microseconds curr_time; if (event.is_completion) { Sim* sim = device_states_[event.node->assigned_device_name()]; --sim->num_running; diff --git a/tensorflow/core/framework/api_def.proto b/tensorflow/core/framework/api_def.proto index 987caee25065d0316bde42a9db75fd4d2a171b8d..98c38efc0e9a8e2ca7caf6b666c8930eb7a32733 100644 --- a/tensorflow/core/framework/api_def.proto +++ b/tensorflow/core/framework/api_def.proto @@ -51,7 +51,8 @@ message ApiDef { // endpoints are deprecated). message Endpoint { // Name should be either like "CamelCaseName" or - // "Package.CamelCaseName". + // "Package.CamelCaseName". Client-language-specific ApiDefs may + // use a snake_case convention instead of CamelCase. string name = 1; // First GraphDef version at which the op is disallowed. @@ -74,7 +75,7 @@ message ApiDef { } repeated Arg in_arg = 4; repeated Arg out_arg = 5; - // List of post-rename in_arg names to specify new argument order. + // List of original in_arg names to specify new argument order. // Length of arg_order should be either empty to keep current order // or match size of in_arg. repeated string arg_order = 11; diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index 4796c3c00a4e791c8c46c4d9290fdb334f04efd0..315c99d32bf855d5f0941f0e5c76bb0548208257 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -1020,6 +1020,29 @@ Status UnknownShape(shape_inference::InferenceContext* c) { return Status::OK(); } +template +Status ReductionShapeHelper(const Tensor* reduction_indices_t, + const int32 input_rank, + std::set& true_indices) { + auto reduction_indices = reduction_indices_t->flat(); + for (int i = 0; i < reduction_indices_t->NumElements(); ++i) { + const T reduction_index = reduction_indices(i); + if (reduction_index < -input_rank || reduction_index >= input_rank) { + return errors::InvalidArgument("Invalid reduction dimension ", + reduction_index, " for input with ", + input_rank, " dimensions."); + } + + auto wrapped_index = reduction_index; + if (wrapped_index < 0) { + wrapped_index += input_rank; + } + + true_indices.insert(wrapped_index); + } + return Status::OK(); +} + Status ReductionShape(InferenceContext* c) { ShapeHandle input = c->input(0); @@ -1050,22 +1073,16 @@ Status ReductionShape(InferenceContext* c) { } const int32 input_rank = c->Rank(input); - std::set true_indices; - auto reduction_indices = reduction_indices_t->flat(); - for (int i = 0; i < reduction_indices_t->NumElements(); ++i) { - int32 reduction_index = reduction_indices(i); - if (reduction_index < -input_rank || reduction_index >= input_rank) { - return errors::InvalidArgument("Invalid reduction dimension ", - reduction_index, " for input with ", - input_rank, " dimensions."); - } - - int32 wrapped_index = reduction_index; - if (wrapped_index < 0) { - wrapped_index += input_rank; - } - - true_indices.insert(wrapped_index); + std::set true_indices; + if (reduction_indices_t->dtype() == DataType::DT_INT32) { + TF_RETURN_IF_ERROR(ReductionShapeHelper(reduction_indices_t, + input_rank, true_indices)); + } else if (reduction_indices_t->dtype() == DataType::DT_INT64) { + TF_RETURN_IF_ERROR(ReductionShapeHelper(reduction_indices_t, + input_rank, true_indices)); + } else { + return errors::InvalidArgument( + "reduction_indices can only be int32 or int64"); } std::vector dims; @@ -1319,11 +1336,10 @@ Status ScatterNdUpdateShape(InferenceContext* c) { Status s = c->Merge(prefix_indices, prefix_updates, &unused); if (!s.ok()) { return errors::InvalidArgument( - "The outer ", num_outer_dims, - " dimensions of indices.shape=", c->DebugString(indices_shape), - " must match the outer ", num_outer_dims, - " dimensions of updates.shape=", c->DebugString(updates_shape), - ": ", s.error_message()); + "The outer ", num_outer_dims, " dimensions of indices.shape=", + c->DebugString(indices_shape), " must match the outer ", + num_outer_dims, " dimensions of updates.shape=", + c->DebugString(updates_shape), ": ", s.error_message()); } ShapeHandle input_suffix; diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h index 14a96c57b58213f5ff8adb75563e125dd5ed173a..33bd5d250cd6b5df8c933e3f353efd9a1eee592c 100644 --- a/tensorflow/core/framework/device_base.h +++ b/tensorflow/core/framework/device_base.h @@ -188,6 +188,9 @@ class DeviceBase { // by GPU devices to return a derived type. virtual PerOpGpuDevice* MakeGpuDevice() { return nullptr; } + virtual DeviceBase* UnderlyingDevice() { return this; } + virtual const DeviceBase* UnderlyingDevice() const { return this; } + // This is overridden by GPU devices to reinitialize the derived // type returned by MakeGpuDevice. virtual void ReinitializeGpuDevice(OpKernelContext* /*context*/, diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h index e8ae9aa74f19fd64f3ff30d28dada883cf02cb5d..305b140a446171ddc4b249c97967057aa3e00152 100644 --- a/tensorflow/core/framework/function.h +++ b/tensorflow/core/framework/function.h @@ -438,6 +438,16 @@ class FunctionLibraryRuntime { // Parameters for remote function execution. bool remote_execution = false; string source_device = ""; // Fully specified device name. + + // Allocator attributes specifying where the args are / rets should be put. + // These should either be {} or match the length of args / retvals. If {}, + // the default allocator attributes will be assumed for all args / retvals. + std::vector args_alloc_attrs; + std::vector rets_alloc_attrs; + + // If true, we create a new IntraProcessRendezvous, else use the existing + // one. + bool create_rendezvous = false; }; typedef std::function DoneCallback; virtual void Run(const Options& opts, Handle handle, diff --git a/tensorflow/core/framework/iterator.proto b/tensorflow/core/framework/iterator.proto new file mode 100644 index 0000000000000000000000000000000000000000..7e5f5ea2e0c2f976855813d2f5e53de0f190872e --- /dev/null +++ b/tensorflow/core/framework/iterator.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package tensorflow; +option cc_enable_arenas = true; +option java_outer_classname = "IteratorProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.util"; + +// Protocol buffer representing the metadata for an iterator's state stored +// as a Variant tensor. +message IteratorStateMetadata { + // A user-specified version string. + string version = 1; + + // Keys for tensors in the VariantTensorDataProto. + repeated string keys = 2; +} diff --git a/tensorflow/core/framework/node_def.proto b/tensorflow/core/framework/node_def.proto index 53aa03108abf3f6fa3240b92bb1c2e703e63cd8b..8fcee32e2986611a1406dfc6998c9e2810b01034 100644 --- a/tensorflow/core/framework/node_def.proto +++ b/tensorflow/core/framework/node_def.proto @@ -35,7 +35,7 @@ message NodeDef { // CONSTRAINT ::= ("job:" JOB_NAME) // | ("replica:" [1-9][0-9]*) // | ("task:" [1-9][0-9]*) - // | ( ("gpu" | "cpu") ":" ([1-9][0-9]* | "*") ) + // | ("device:" [A-Za-z]* ":" ([1-9][0-9]* | "*") ) // // Valid values for this string include: // * "/job:worker/replica:0/task:1/device:GPU:3" (full specification) diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc index 143da996a1e2144a44101b004ea4130abb8e5af6..1e93e9be0955c9d62588e009e5a6d899ce33698d 100644 --- a/tensorflow/core/framework/op_gen_lib.cc +++ b/tensorflow/core/framework/op_gen_lib.cc @@ -17,11 +17,12 @@ limitations under the License. #include #include "tensorflow/core/framework/attr_value.pb.h" -#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/framework/op_gen_overrides.pb.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/protobuf.h" namespace tensorflow { @@ -393,4 +394,221 @@ const OpGenOverride* OpGenOverrideMap::ApplyOverride(OpDef* op_def) const { return &proto; } +namespace { + +// Initializes given ApiDef with data in OpDef. +void InitApiDefFromOpDef(const OpDef& op_def, ApiDef* api_def) { + api_def->set_graph_op_name(op_def.name()); + api_def->set_visibility(ApiDef::VISIBLE); + + auto* endpoint = api_def->add_endpoint(); + endpoint->set_name(op_def.name()); + if (op_def.has_deprecation()) { + endpoint->set_deprecation_version(op_def.deprecation().version()); + } + + for (const auto& op_in_arg : op_def.input_arg()) { + auto* api_in_arg = api_def->add_in_arg(); + api_in_arg->set_name(op_in_arg.name()); + api_in_arg->set_rename_to(op_in_arg.name()); + api_in_arg->set_description(op_in_arg.description()); + + *api_def->add_arg_order() = op_in_arg.name(); + } + for (const auto& op_out_arg : op_def.output_arg()) { + auto* api_out_arg = api_def->add_out_arg(); + api_out_arg->set_name(op_out_arg.name()); + api_out_arg->set_rename_to(op_out_arg.name()); + api_out_arg->set_description(op_out_arg.description()); + } + for (const auto& op_attr : op_def.attr()) { + auto* api_attr = api_def->add_attr(); + api_attr->set_name(op_attr.name()); + api_attr->set_rename_to(op_attr.name()); + if (op_attr.has_default_value()) { + *api_attr->mutable_default_value() = op_attr.default_value(); + } + api_attr->set_description(op_attr.description()); + } + api_def->set_summary(op_def.summary()); + api_def->set_description(op_def.description()); +} + +// Updates base_arg based on overrides in new_arg. +void MergeArg(ApiDef::Arg* base_arg, const ApiDef::Arg& new_arg) { + if (!new_arg.rename_to().empty()) { + base_arg->set_rename_to(new_arg.rename_to()); + } + if (!new_arg.description().empty()) { + base_arg->set_description(new_arg.description()); + } +} + +// Updates base_attr based on overrides in new_attr. +void MergeAttr(ApiDef::Attr* base_attr, const ApiDef::Attr& new_attr) { + if (!new_attr.rename_to().empty()) { + base_attr->set_rename_to(new_attr.rename_to()); + } + if (new_attr.has_default_value()) { + *base_attr->mutable_default_value() = new_attr.default_value(); + } + if (!new_attr.description().empty()) { + base_attr->set_description(new_attr.description()); + } +} + +// Updates base_api_def based on overrides in new_api_def. +Status MergeApiDefs(ApiDef* base_api_def, const ApiDef& new_api_def) { + // Merge visibility + if (new_api_def.visibility() != ApiDef::DEFAULT_VISIBILITY) { + base_api_def->set_visibility(new_api_def.visibility()); + } + // Merge endpoints + if (new_api_def.endpoint_size() > 0) { + base_api_def->clear_endpoint(); + std::copy( + new_api_def.endpoint().begin(), new_api_def.endpoint().end(), + protobuf::RepeatedFieldBackInserter(base_api_def->mutable_endpoint())); + } + // Merge args + for (const auto& new_arg : new_api_def.in_arg()) { + bool found_base_arg = false; + for (int i = 0; i < base_api_def->in_arg_size(); ++i) { + auto* base_arg = base_api_def->mutable_in_arg(i); + if (base_arg->name() == new_arg.name()) { + MergeArg(base_arg, new_arg); + found_base_arg = true; + break; + } + } + if (!found_base_arg) { + return errors::FailedPrecondition("Argument ", new_arg.name(), + " not defined in base api for ", + base_api_def->graph_op_name()); + } + } + for (const auto& new_arg : new_api_def.out_arg()) { + bool found_base_arg = false; + for (int i = 0; i < base_api_def->out_arg_size(); ++i) { + auto* base_arg = base_api_def->mutable_out_arg(i); + if (base_arg->name() == new_arg.name()) { + MergeArg(base_arg, new_arg); + found_base_arg = true; + break; + } + } + if (!found_base_arg) { + return errors::FailedPrecondition("Argument ", new_arg.name(), + " not defined in base api for ", + base_api_def->graph_op_name()); + } + } + // Merge arg order + if (new_api_def.arg_order_size() > 0) { + // Validate that new arg_order is correct. + if (new_api_def.arg_order_size() != base_api_def->arg_order_size()) { + return errors::FailedPrecondition( + "Invalid number of arguments ", new_api_def.arg_order_size(), " for ", + base_api_def->graph_op_name(), + ". Expected: ", base_api_def->arg_order_size()); + } + if (!std::is_permutation(new_api_def.arg_order().begin(), + new_api_def.arg_order().end(), + base_api_def->arg_order().begin())) { + return errors::FailedPrecondition( + "Invalid arg_order: ", str_util::Join(new_api_def.arg_order(), ", "), + " for ", base_api_def->graph_op_name(), + ". All elements in arg_order override must match base arg_order: ", + str_util::Join(base_api_def->arg_order(), ", ")); + } + base_api_def->clear_arg_order(); + std::copy( + new_api_def.arg_order().begin(), new_api_def.arg_order().end(), + protobuf::RepeatedFieldBackInserter(base_api_def->mutable_arg_order())); + } + // Merge attributes + for (const auto& new_attr : new_api_def.attr()) { + bool found_base_attr = false; + for (int i = 0; i < base_api_def->attr_size(); ++i) { + auto* base_attr = base_api_def->mutable_attr(i); + if (base_attr->name() == new_attr.name()) { + MergeAttr(base_attr, new_attr); + found_base_attr = true; + break; + } + } + if (!found_base_attr) { + return errors::FailedPrecondition("Attribute ", new_attr.name(), + " not defined in base api for ", + base_api_def->graph_op_name()); + } + } + // Merge summary + if (!new_api_def.summary().empty()) { + base_api_def->set_summary(new_api_def.summary()); + } + // Merge description + auto description = new_api_def.description().empty() + ? base_api_def->description() + : new_api_def.description(); + + if (!new_api_def.description_prefix().empty()) { + description = + strings::StrCat(new_api_def.description_prefix(), "\n", description); + } + if (!new_api_def.description_suffix().empty()) { + description = + strings::StrCat(description, "\n", new_api_def.description_suffix()); + } + base_api_def->set_description(description); + return Status::OK(); +} +} // namespace + +ApiDefMap::ApiDefMap(const OpList& op_list) { + for (const auto& op : op_list.op()) { + ApiDef api_def; + InitApiDefFromOpDef(op, &api_def); + map_[op.name()] = api_def; + } +} + +ApiDefMap::~ApiDefMap() {} + +Status ApiDefMap::LoadFileList(Env* env, const std::vector& filenames) { + for (const auto& filename : filenames) { + TF_RETURN_IF_ERROR(LoadFile(env, filename)); + } + return Status::OK(); +} + +Status ApiDefMap::LoadFile(Env* env, const string& filename) { + if (filename.empty()) return Status::OK(); + string contents; + TF_RETURN_IF_ERROR(ReadFileToString(env, filename, &contents)); + TF_RETURN_IF_ERROR(LoadApiDef(contents)); + return Status::OK(); +} + +Status ApiDefMap::LoadApiDef(const string& api_def_file_contents) { + const string contents = PBTxtFromMultiline(api_def_file_contents); + ApiDefs api_defs; + protobuf::TextFormat::ParseFromString(contents, &api_defs); + for (const auto& api_def : api_defs.op()) { + // Check if the op definition is already loaded. + if (map_.find(api_def.graph_op_name()) != map_.end()) { + // Overwrite current api def with data in api_def. + TF_RETURN_IF_ERROR(MergeApiDefs(&map_[api_def.graph_op_name()], api_def)); + } else { + return errors::FailedPrecondition( + "Unexpected ApiDef override: ", api_def.graph_op_name(), + " is not defined in base ApiDef."); + } + } + return Status::OK(); +} + +const tensorflow::ApiDef* ApiDefMap::GetApiDef(const string& name) const { + return gtl::FindOrNull(map_, name); +} } // namespace tensorflow diff --git a/tensorflow/core/framework/op_gen_lib.h b/tensorflow/core/framework/op_gen_lib.h index dbe0a8e190431bc04ceeef917fc7ea4ceca8e73d..efb287477bedde9bfbdef8e318bf6804e79f1ac5 100644 --- a/tensorflow/core/framework/op_gen_lib.h +++ b/tensorflow/core/framework/op_gen_lib.h @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include "tensorflow/core/framework/api_def.pb.h" +#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/platform/env.h" @@ -74,6 +76,48 @@ class OpGenOverrideMap { std::unordered_map> map_; }; +// Takes a list of files with ApiDefs text protos, and allows you to +// look up the specific ApiDef for any given op. +class ApiDefMap { + public: + // OpList must be a superset of ops of any subsequently loaded + // ApiDef. + explicit ApiDefMap(const OpList& op_list); + ~ApiDefMap(); + + // You can call this method multiple times to load multiple + // sets of files. Api definitions are merged if the same + // op definition is loaded multiple times. Later-loaded + // definitions take precedense. + // ApiDefs loaded from files must contain a subset of ops defined + // in the OpList passed to the constructor. + Status LoadFileList(Env* env, const std::vector& filenames); + + // Load a single file. Api definitions are merged if the same + // op definition is loaded multiple times. Later-loaded + // definitions take precedense. + // ApiDefs loaded from file must contain a subset of ops defined + // in the OpList passed to the constructor. + Status LoadFile(Env* env, const string& filename); + + // Load ApiDefs from string containing ApiDefs text proto. + // api_def_file_contents is expected to be in "multiline format". + // ApiDefs must contain a subset of ops defined in OpsList + // passed to the constructor. + Status LoadApiDef(const string& api_def_file_contents); + + // Look up ApiDef proto based on the given graph op name. + // If graph op name is not in this ApiDefMap, returns nullptr. + // + // Note: Returned ApiDef pointer should stay valid even after calling + // Load* functions defined above. Subsequent calls to Load* might modify + // returned ApiDef contents, but should never remove the ApiDef itself. + const ApiDef* GetApiDef(const string& name) const; + + private: + std::unordered_map map_; +}; + } // namespace tensorflow #endif // TENSORFLOW_FRAMEWORK_OP_GEN_LIB_H_ diff --git a/tensorflow/core/framework/op_gen_lib_test.cc b/tensorflow/core/framework/op_gen_lib_test.cc index cc1d117f38466f02fc83c95384f6f13449e46fb3..da9b4dfbb1738c855c0bfc4752853d5d501d80a8 100644 --- a/tensorflow/core/framework/op_gen_lib_test.cc +++ b/tensorflow/core/framework/op_gen_lib_test.cc @@ -15,11 +15,60 @@ limitations under the License. #include "tensorflow/core/framework/op_gen_lib.h" +#include "tensorflow/core/framework/op_def.pb.h" #include "tensorflow/core/platform/test.h" namespace tensorflow { namespace { +constexpr char kTestOpList[] = R"(op { + name: "testop" + input_arg { + name: "arg_a" + } + input_arg { + name: "arg_b" + } + output_arg { + name: "arg_c" + } + attr { + name: "attr_a" + } + deprecation { + version: 123 + explanation: "foo" + } +)"; + +constexpr char kTestApiDef[] = R"(op { + graph_op_name: "testop" + visibility: VISIBLE + endpoint { + name: "testop1" + } + in_arg { + name: "arg_a" + } + in_arg { + name: "arg_b" + } + out_arg { + name: "arg_c" + } + attr { + name: "attr_a" + } + summary: "Mock op for testing." + description: <DebugString()); +} + +TEST(OpGenLibTest, ApiDefLoadSingleApiDef) { + const string expected_api_def = R"(op { + graph_op_name: "testop" + visibility: VISIBLE + endpoint { + name: "testop1" + } + in_arg { + name: "arg_a" + rename_to: "arg_a" + } + in_arg { + name: "arg_b" + rename_to: "arg_b" + } + out_arg { + name: "arg_c" + rename_to: "arg_c" + } + attr { + name: "attr_a" + rename_to: "attr_a" + } + summary: "Mock op for testing." + description: "Description for the\ntestop." + arg_order: "arg_a" + arg_order: "arg_b" +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + const auto* api_def = api_map.GetApiDef("testop"); + EXPECT_EQ(1, api_def->endpoint_size()); + EXPECT_EQ("testop1", api_def->endpoint(0).name()); + + ApiDefs api_defs; + *api_defs.add_op() = *api_def; + EXPECT_EQ(expected_api_def, api_defs.DebugString()); +} + +TEST(OpGenLibTest, ApiDefOverrideVisibility) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + endpoint { + name: "testop2" + } +} +)"; + const string api_def2 = R"( +op { + graph_op_name: "testop" + visibility: HIDDEN + endpoint { + name: "testop2" + } +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + auto* api_def = api_map.GetApiDef("testop"); + EXPECT_EQ(ApiDef::VISIBLE, api_def->visibility()); + + // Loading ApiDef with default visibility should + // keep current visibility. + TF_CHECK_OK(api_map.LoadApiDef(api_def1)); + EXPECT_EQ(ApiDef::VISIBLE, api_def->visibility()); + + // Loading ApiDef with non-default visibility, + // should update visibility. + TF_CHECK_OK(api_map.LoadApiDef(api_def2)); + EXPECT_EQ(ApiDef::HIDDEN, api_def->visibility()); +} + +TEST(OpGenLibTest, ApiDefOverrideEndpoints) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + endpoint { + name: "testop2" + } +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + auto* api_def = api_map.GetApiDef("testop"); + ASSERT_EQ(1, api_def->endpoint_size()); + EXPECT_EQ("testop1", api_def->endpoint(0).name()); + + TF_CHECK_OK(api_map.LoadApiDef(api_def1)); + ASSERT_EQ(1, api_def->endpoint_size()); + EXPECT_EQ("testop2", api_def->endpoint(0).name()); +} + +TEST(OpGenLibTest, ApiDefOverrideArgs) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + in_arg { + name: "arg_a" + rename_to: "arg_aa" + } + out_arg { + name: "arg_c" + rename_to: "arg_cc" + } + arg_order: "arg_b" + arg_order: "arg_a" +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + TF_CHECK_OK(api_map.LoadApiDef(api_def1)); + const auto* api_def = api_map.GetApiDef("testop"); + ASSERT_EQ(2, api_def->in_arg_size()); + EXPECT_EQ("arg_aa", api_def->in_arg(0).rename_to()); + // 2nd in_arg is not renamed + EXPECT_EQ("arg_b", api_def->in_arg(1).rename_to()); + + ASSERT_EQ(1, api_def->out_arg_size()); + EXPECT_EQ("arg_cc", api_def->out_arg(0).rename_to()); + + ASSERT_EQ(2, api_def->arg_order_size()); + EXPECT_EQ("arg_b", api_def->arg_order(0)); + EXPECT_EQ("arg_a", api_def->arg_order(1)); +} + +TEST(OpGenLibTest, ApiDefOverrideDescriptions) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + summary: "New summary" + description: <summary()); + EXPECT_EQ("A\nNew description\nZ", api_def->description()); + EXPECT_EQ("", api_def->description_prefix()); + EXPECT_EQ("", api_def->description_suffix()); + + TF_CHECK_OK(api_map.LoadApiDef(api_def2)); + EXPECT_EQ("B\nA\nNew description\nZ\nY", api_def->description()); + EXPECT_EQ("", api_def->description_prefix()); + EXPECT_EQ("", api_def->description_suffix()); +} + +TEST(OpGenLibTest, ApiDefInvalidOpInOverride) { + const string api_def1 = R"( +op { + graph_op_name: "different_testop" + endpoint { + name: "testop2" + } +} +)"; + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + auto status = api_map.LoadApiDef(api_def1); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); +} + +TEST(OpGenLibTest, ApiDefInvalidArgOrder) { + const string api_def1 = R"( +op { + graph_op_name: "testop" + arg_order: "arg_a" + arg_order: "unexpected_arg" +} +)"; + + const string api_def2 = R"( +op { + graph_op_name: "testop" + arg_order: "arg_a" +} +)"; + + const string api_def3 = R"( +op { + graph_op_name: "testop" + arg_order: "arg_a" + arg_order: "arg_a" +} +)"; + + OpList op_list; + protobuf::TextFormat::ParseFromString(kTestOpList, &op_list); // NOLINT + ApiDefMap api_map(op_list); + TF_CHECK_OK(api_map.LoadApiDef(kTestApiDef)); + + // Loading with incorrect arg name in arg_order should fail. + auto status = api_map.LoadApiDef(api_def1); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); + + // Loading with incorrect number of args in arg_order should fail. + status = api_map.LoadApiDef(api_def2); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); + + // Loading with the same argument twice in arg_order should fail. + status = api_map.LoadApiDef(api_def3); + ASSERT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code()); +} } // namespace } // namespace tensorflow diff --git a/tensorflow/core/framework/register_types.h b/tensorflow/core/framework/register_types.h index 61e722e57b7e8e29e5edab699d5c39b8a6be716b..c31ab18cc12f699d9295b0688e59db775be6b5d8 100644 --- a/tensorflow/core/framework/register_types.h +++ b/tensorflow/core/framework/register_types.h @@ -87,7 +87,7 @@ limitations under the License. #elif defined(__ANDROID_TYPES_FULL__) -// Only half, float, int32, int64, and quantized types are supported. +// Only half, float, int32, int64, bool, and quantized types are supported. #define TF_CALL_float(m) m(float) #define TF_CALL_double(m) #define TF_CALL_int32(m) m(::tensorflow::int32) @@ -117,7 +117,7 @@ limitations under the License. #else // defined(IS_MOBILE_PLATFORM) && !defined(__ANDROID_TYPES_FULL__) -// Only float and int32 are supported. +// Only float, int32, and bool are supported. #define TF_CALL_float(m) m(float) #define TF_CALL_double(m) #define TF_CALL_int32(m) m(::tensorflow::int32) diff --git a/tensorflow/core/framework/types.cc b/tensorflow/core/framework/types.cc index cc86871caeda9822e92124d4151d47163fe9025c..faae19585d9dd2bc5f351772af93723daaa3b8be 100644 --- a/tensorflow/core/framework/types.cc +++ b/tensorflow/core/framework/types.cc @@ -335,6 +335,18 @@ bool DataTypeIsInteger(DataType dt) { } } +bool DataTypeIsUnsigned(DataType dt) { + switch (dt) { + case DT_UINT8: + case DT_UINT16: + case DT_UINT32: + case DT_UINT64: + return true; + default: + return false; + } +} + int DataTypeSize(DataType dt) { #define CASE(T) \ case DataTypeToEnum::value: \ diff --git a/tensorflow/core/framework/types.h b/tensorflow/core/framework/types.h index 300a57e948f979f52ef2b8d6a875e3d8b1c860df..dc53ed41780d90448872b1bd98e97f5e16d49592 100644 --- a/tensorflow/core/framework/types.h +++ b/tensorflow/core/framework/types.h @@ -227,6 +227,9 @@ bool DataTypeIsQuantized(DataType dt); // Is the dtype nonquantized integral? bool DataTypeIsInteger(DataType dt); +// Is the dtype an unsigned integral type? +bool DataTypeIsUnsigned(DataType dt); + // Returns a 0 on failure int DataTypeSize(DataType dt); diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc index 15f7b9fe8c6b6b25d8e863576409e828bee8acc8..8fe4f535fbb8c1a93fd06c5858ad2095d50f6808 100644 --- a/tensorflow/core/graph/graph_constructor.cc +++ b/tensorflow/core/graph/graph_constructor.cc @@ -79,6 +79,7 @@ class GraphConstructor { skip_mapped_nodes(in.skip_mapped_nodes), control_dependencies(in.control_dependencies), return_tensors(in.return_tensors), + return_nodes(in.return_nodes), importing(true) {} bool allow_internal_ops; @@ -89,6 +90,7 @@ class GraphConstructor { bool skip_mapped_nodes; std::vector control_dependencies; std::vector return_tensors; + std::vector return_nodes; // TODO(ashankar): This bool exists to separate out functionality required // to make ImportGraphDef a close equivalent of Python's import_graph_def @@ -109,6 +111,7 @@ class GraphConstructor { const FunctionDefLibrary* library, Graph* g, ShapeRefiner* refiner, std::vector>* return_tensors, + std::vector* return_nodes, std::vector* unused_input_map_keys) { if (versions) { TF_RETURN_IF_ERROR(CheckVersions(*versions, TF_GRAPH_DEF_VERSION, @@ -116,7 +119,7 @@ class GraphConstructor { "GraphDef", "graph")); } GraphConstructor c(opts, node_defs, versions, library, g, refiner, - return_tensors, unused_input_map_keys); + return_tensors, return_nodes, unused_input_map_keys); const Status s = c.TryImport(); if (!s.ok()) c.Undo(); return s; @@ -128,6 +131,7 @@ class GraphConstructor { const FunctionDefLibrary* library, Graph* g, ShapeRefiner* refiner, std::vector>* return_tensors, + std::vector* return_nodes, std::vector* unused_input_map_keys) : opts_(opts), node_defs_(node_defs), @@ -137,6 +141,7 @@ class GraphConstructor { original_versions_(g->versions()), refiner_(refiner), return_tensors_(return_tensors), + return_nodes_(return_nodes), unused_input_map_keys_(unused_input_map_keys) {} Status TryImport() { @@ -148,6 +153,7 @@ class GraphConstructor { TF_RETURN_IF_ERROR(AddBackEdges()); TF_RETURN_IF_ERROR(UpdateVersionDef()); TF_RETURN_IF_ERROR(PopulateReturnTensors()); + TF_RETURN_IF_ERROR(PopulateReturnNodes()); FixupSourceAndSinkEdges(g_); return Status::OK(); } @@ -160,6 +166,7 @@ class GraphConstructor { Status AddBackEdges(); Status UpdateVersionDef(); Status PopulateReturnTensors(); + Status PopulateReturnNodes(); void Undo(); @@ -196,6 +203,9 @@ class GraphConstructor { // May be null. Not owned. std::vector>* return_tensors_; + // May be null. Not owned. + std::vector* return_nodes_; + // May be null. Not owned. std::vector* unused_input_map_keys_; @@ -836,9 +846,10 @@ Status GraphConstructor::Convert() { } } - // TODO(skyewm): remove conditional when b/35715995 ("Functions lack shape - // inference") is resolved. - if (g_->flib_def().Find(node_def->name()) == nullptr) { + // Function shape inference is supported on an opt-in basis per + // ShapeRefiner. + if (refiner_->function_shape_inference_supported() || + g_->flib_def().Find(node_def->name()) == nullptr) { TF_RETURN_IF_ERROR(ValidateShape(node)); } @@ -913,7 +924,8 @@ Status GraphConstructor::PopulateReturnTensors() { // Locate id in imported nodes auto iter = gdef_nodes_.find(id.first); if (iter == gdef_nodes_.end()) { - return errors::InvalidArgument("Requested return node '", id.first, + return errors::InvalidArgument("Requested return tensor '", + id.ToString(), "' not found in graph def"); } int num_outputs = iter->second.node->num_outputs(); @@ -935,6 +947,19 @@ Status GraphConstructor::PopulateReturnTensors() { return Status::OK(); } +Status GraphConstructor::PopulateReturnNodes() { + if (opts_.return_nodes.empty()) return Status::OK(); + for (StringPiece name : opts_.return_nodes) { + auto iter = gdef_nodes_.find(name); + if (iter == gdef_nodes_.end()) { + return errors::InvalidArgument("Requested return node '", name, + "' not found in graph def"); + } + return_nodes_->push_back(iter->second.node); + } + return Status::OK(); +} + void GraphConstructor::Undo() { for (const auto& iter : gdef_nodes_) { if (iter.second.node != nullptr) { @@ -965,7 +990,8 @@ Status ConvertGraphDefToGraph(const GraphConstructorOptions& opts, ShapeRefiner refiner(gdef.versions().producer(), g->op_registry()); return GraphConstructor::Construct( opts, gdef.node(), &gdef.versions(), &gdef.library(), g, &refiner, - /*return_tensors=*/nullptr, /*unused_input_map_keys=*/nullptr); + /*return_tensors=*/nullptr, /*return_nodes=*/nullptr, + /*unused_input_map_keys=*/nullptr); } Status ConvertNodeDefsToGraph(const GraphConstructorOptions& opts, @@ -978,31 +1004,40 @@ Status ConvertNodeDefsToGraph(const GraphConstructorOptions& opts, } return GraphConstructor::Construct(opts, node_defs, nullptr, nullptr, g, &refiner, /*return_tensors=*/nullptr, + /*return_nodes=*/nullptr, /*unused_input_map_keys=*/nullptr); } Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef, Graph* g, ShapeRefiner* refiner, - std::vector>* return_tensors, - std::vector* unused_input_map_keys) { + ImportGraphDefResults* results) { if (!opts.return_tensors.empty()) { - if (return_tensors == nullptr) { + if (results == nullptr) { return errors::InvalidArgument( - "return_tensors argument to ImportGraphDef() must be non-null if " + "results argument to ImportGraphDef() must be non-null if " "opts.return_tensors is non-empty"); } - if (!return_tensors->empty()) { + } + + if (!opts.return_nodes.empty()) { + if (opts.skip_mapped_nodes) { return errors::InvalidArgument( - "return_tensors argument to ImportGraphDef() should be empty (has " - "size ", - return_tensors->size(), ")"); + "Requesting return_nodes with skip_mapped_nodes set is not currently " + "supported"); + } + if (results == nullptr) { + return errors::InvalidArgument( + "results argument to ImportGraphDef() must be non-null if " + "opts.return_nodes is non-empty"); } } - if (unused_input_map_keys != nullptr && !unused_input_map_keys->empty()) { - return errors::InvalidArgument( - "If non-null, unused_input_map_keys argument to ImportGraphDef() should" - " be empty (has size ", - unused_input_map_keys->size(), ")"); + + if (results != nullptr) { + if (!results->return_tensors.empty() || !results->return_nodes.empty() || + !results->unused_input_map_keys.empty()) { + return errors::InvalidArgument( + "All fields in results argument to ImportGraphDef() must be empty."); + } } ShapeRefiner default_refiner(gdef.versions().producer(), g->op_registry()); @@ -1034,9 +1069,16 @@ Status ImportGraphDef(const ImportGraphDefOptions& opts, const GraphDef& gdef, refiner->set_graph_def_version( std::min(refiner->graph_def_version(), gdef.versions().producer())); - return GraphConstructor::Construct(opts, gdef.node(), &gdef.versions(), - &gdef.library(), g, refiner, - return_tensors, unused_input_map_keys); + if (results == nullptr) { + return GraphConstructor::Construct(opts, gdef.node(), &gdef.versions(), + &gdef.library(), g, refiner, nullptr, + nullptr, nullptr); + } else { + return GraphConstructor::Construct( + opts, gdef.node(), &gdef.versions(), &gdef.library(), g, refiner, + &results->return_tensors, &results->return_nodes, + &results->unused_input_map_keys); + } } void CopyGraph(const Graph& src, Graph* dest) { diff --git a/tensorflow/core/graph/graph_constructor.h b/tensorflow/core/graph/graph_constructor.h index a8f9f2b24596eca02cb859c0e52e1edd57bf1027..a3644788788544728193e4f648fa562e1275ffdc 100644 --- a/tensorflow/core/graph/graph_constructor.h +++ b/tensorflow/core/graph/graph_constructor.h @@ -72,8 +72,6 @@ struct ImportGraphDefOptions { // used to create the existing nodes referenced in `input_map`. // TODO(skyewm): can we remove this requirement? How do we access the original // shape refiner? - // - // TODO(skyewm): add functionality to retrieve unused `input_map` keys std::map input_map; // If true, nodes that will have all output edges removed because of @@ -88,10 +86,10 @@ struct ImportGraphDefOptions { // other nodes in `gdef`. std::vector control_dependencies; - // Tensors in `gdef` that will be returned via the `return_tensors` output - // parameter of `ImportGraphDef()`. If this list is non-empty, the caller must - // pass an empty vector to `ImportGraphDef()`. The vector will be populated - // with the imported nodes in `g`. + // Tensors in `gdef` that will be returned via the ImportGraphDefResults + // output parameter of `ImportGraphDef()`. If this list is non-empty, the + // caller must pass a results object to `ImportGraphDef()`. The + // `return_tensors` field will be populated with the imported nodes in `g`. // // Entries should not include `prefix`, i.e., each TensorId's name should be // the name as it originally appears in `gdef`. @@ -100,12 +98,43 @@ struct ImportGraphDefOptions { // corresponding existing tensor in `g` will be returned. std::vector return_tensors; + // The names of nodes in `gdef` that will be returned via the + // ImportGraphDefResults output parameter of `ImportGraphDef()`. If this list + // is non-empty, the caller must pass a results object to + // `ImportGraphDef()`. The `return_nodes` field will be populated with the + // imported nodes in `g`. + // + // Entries should not include `prefix`, i.e., each node's name should be the + // name as it originally appears in `gdef`. + // + // Unlike `return_tensors`, `input_map` has no effect on the nodes + // returned. `return_nodes` must be empty if `skip_mapped_nodes` is true. + // TODO(skyewm): make this work with `skip_mapped_nodes` if there's a need. + std::vector return_nodes; + // TODO(ashankar): Enable handling of GraphDefs produced by newer binaries // with ops that are not defined in the binary calling ImportGraphDef. // Similar to the producer_op_list argument to import_graph_def in the // python API. }; +// Optional results that may be returned by ImportGraphDef. +struct ImportGraphDefResults { + // The requested tensors associated with + // ImportGraphDefOptions::return_tensors. Note that the index may be different + // than the requested index if the returned tensor has been remapped according + // to `input_map`. + typedef int Index; + std::vector> return_tensors; + + // The requested nodes associated with ImportGraphDefOptions::return_nodes. + std::vector return_nodes; + + // Keys in ImportGraphDefOptions::input_map that weren't used as an input to + // any node in`gdef`. + std::vector unused_input_map_keys; +}; + // Adds the graph in GraphDef `gdef` into an existing Graph `*g`. // // On error, returns non-OK and leaves `*g` unmodified. @@ -115,21 +144,16 @@ struct ImportGraphDefOptions { // allows the caller to validate shapes of those nodes (since // ShapeRefiner::AddNode must be called in topological order). // -// Each `return_tensors` entry is the requested node and output index. The index -// is included in case the returned tensor has been remapped according to -// `input_map`. -// -// If `unused_input_map_keys` is non-null, it should be empty and will be -// populated with any keys in `opts.input_map` that aren't used as an input to -// any node in `gdef`. +// `results` must be non-null if `opts.return_tensors` or `opts.result_nodes` is +// non-empty. It can also be set to fetch the unused input map keys. If it's +// non-null, all the vector fields must be empty. // // TODO(ashankar): Push this mechanism and get rid of Session::Extend() // as a means of enhancing an existing Graph. -extern Status ImportGraphDef( - const ImportGraphDefOptions& opts, const GraphDef& gdef, Graph* g, - ShapeRefiner* refiner, - std::vector>* return_tensors = nullptr, - std::vector* unused_input_map_keys = nullptr); +extern Status ImportGraphDef(const ImportGraphDefOptions& opts, + const GraphDef& gdef, Graph* g, + ShapeRefiner* refiner, + ImportGraphDefResults* results = nullptr); // Make a copy of "src" into "*dest". // diff --git a/tensorflow/core/graph/graph_constructor_test.cc b/tensorflow/core/graph/graph_constructor_test.cc index f88d707ec5ea216b7dc85d241b3df569e4a2facc..5242c56ce6de63fbe1d03e596cc6123844be4a50 100644 --- a/tensorflow/core/graph/graph_constructor_test.cc +++ b/tensorflow/core/graph/graph_constructor_test.cc @@ -71,14 +71,12 @@ class GraphConstructorTest : public ::testing::Test { void ExpectError(const string& gdef_ascii, const ImportGraphDefOptions& opts, const std::vector& expected_error_strs, ShapeRefiner* refiner = nullptr, - std::vector>* return_tensors = nullptr, - std::vector* unused_input_map_keys = nullptr) { + ImportGraphDefResults* results = nullptr) { // Used to verify that errors don't change graph const string original_graph_description = GraphDebugString(); Convert(gdef_ascii); - Status status = ImportGraphDef(opts, gdef_, &graph_, refiner, - return_tensors, unused_input_map_keys); + Status status = ImportGraphDef(opts, gdef_, &graph_, refiner, results); EXPECT_FALSE(status.ok()); for (const string& error : expected_error_strs) { @@ -97,11 +95,9 @@ class GraphConstructorTest : public ::testing::Test { void ExpectOK(const string& gdef_ascii, const ImportGraphDefOptions& opts, ShapeRefiner* refiner = nullptr, - std::vector>* return_tensors = nullptr, - std::vector* unused_input_map_keys = nullptr) { + ImportGraphDefResults* results = nullptr) { Convert(gdef_ascii); - Status s = ImportGraphDef(opts, gdef_, &graph_, refiner, return_tensors, - unused_input_map_keys); + Status s = ImportGraphDef(opts, gdef_, &graph_, refiner, results); EXPECT_EQ(Status::OK(), s) << s; } @@ -1440,26 +1436,25 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapDuplicateNodeNames) { TEST_F(GraphConstructorTest, ImportGraphDef_InputMapUnusedKeys) { ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); - std::vector unused_input_map_keys; - // No input map ImportGraphDefOptions opts; + ImportGraphDefResults results; ExpectOK( "node { name: 'W1' op: 'TestParams' }" "node { name: 'input' op: 'TestInput' }", - opts, &refiner, nullptr, &unused_input_map_keys); - EXPECT_TRUE(unused_input_map_keys.empty()); + opts, &refiner, &results); + EXPECT_TRUE(results.unused_input_map_keys.empty()); // Non-empty unused_input_map_keys - unused_input_map_keys.push_back(TensorId()); - ExpectError("node { name: 'W2' op: 'TestParams' }", opts, - {"If non-null, unused_input_map_keys argument to ImportGraphDef()" - " should be empty (has size 1)"}, - &refiner, nullptr, &unused_input_map_keys); + results.unused_input_map_keys.push_back(TensorId()); + ExpectError( + "node { name: 'W2' op: 'TestParams' }", opts, + {"All fields in results argument to ImportGraphDef() must be empty."}, + &refiner, &results); // Input map with some used, some unused keys const int kControlSlot = Graph::kControlSlot; - unused_input_map_keys.clear(); + results.unused_input_map_keys.clear(); opts.input_map[TensorId("W2", kControlSlot)] = TensorId("W1", kControlSlot); opts.input_map[TensorId("new_input", 0)] = TensorId("input", 0); opts.input_map[TensorId("new_input", 1)] = TensorId("input", 0); @@ -1473,11 +1468,11 @@ TEST_F(GraphConstructorTest, ImportGraphDef_InputMapUnusedKeys) { node { name: 't1' op: 'TestMul' input: [ 'new_input:0', 'new_input:1' ] } node { name: 't2' op: 'TestMul' input: [ 't1:0', 't1:0' ] } )EOF", - opts, &refiner, nullptr, &unused_input_map_keys); + opts, &refiner, &results); std::vector expected_unused_keys = { TensorId("new_input", kControlSlot), TensorId("t1", 1)}; - EXPECT_EQ(unused_input_map_keys, expected_unused_keys); + EXPECT_EQ(results.unused_input_map_keys, expected_unused_keys); } TEST_F(GraphConstructorTest, ImportGraphDef_SkipMappedNodes_FullyMapped) { @@ -1567,11 +1562,11 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensors) { opts.return_tensors.push_back({"input", 1}); opts.return_tensors.push_back({"t1", 0}); opts.return_tensors.push_back({"input", 0}); - std::vector> return_tensors; + ImportGraphDefResults results; ExpectOK( "node { name: 'input' op: 'TestInput' }" "node { name: 't1' op: 'TestMul' input: ['input:0', 'input:1'] }", - opts, &refiner, &return_tensors); + opts, &refiner, &results); // Sanity checks EXPECT_TRUE(HasNode("input")); @@ -1580,74 +1575,70 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensors) { EXPECT_TRUE(HasEdge("input", 1, "t1", 1)); // Check return tensors - ASSERT_EQ(return_tensors.size(), 3); - EXPECT_EQ(return_tensors[0].first->name(), "input"); - EXPECT_EQ(return_tensors[0].second, 1); - EXPECT_EQ(return_tensors[1].first->name(), "t1"); - EXPECT_EQ(return_tensors[1].second, 0); - EXPECT_EQ(return_tensors[2].first->name(), "input"); - EXPECT_EQ(return_tensors[2].second, 0); + ASSERT_EQ(results.return_tensors.size(), 3); + EXPECT_EQ(results.return_tensors[0].first->name(), "input"); + EXPECT_EQ(results.return_tensors[0].second, 1); + EXPECT_EQ(results.return_tensors[1].first->name(), "t1"); + EXPECT_EQ(results.return_tensors[1].second, 0); + EXPECT_EQ(results.return_tensors[2].first->name(), "input"); + EXPECT_EQ(results.return_tensors[2].second, 0); // Test using prefix and returning element from input_map opts.return_tensors.clear(); - return_tensors.clear(); + results = ImportGraphDefResults(); opts.prefix = "import"; opts.input_map[{"new_input", 1}] = {"input", 0}; opts.return_tensors.push_back({"new_input", 0}); opts.return_tensors.push_back({"new_input", 1}); ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner, - &return_tensors); + &results); EXPECT_TRUE(HasNode("import/new_input")); - ASSERT_EQ(return_tensors.size(), 2); - EXPECT_EQ(return_tensors[0].first->name(), "import/new_input"); - EXPECT_EQ(return_tensors[0].second, 0); - EXPECT_EQ(return_tensors[1].first->name(), "input"); - EXPECT_EQ(return_tensors[1].second, 0); + ASSERT_EQ(results.return_tensors.size(), 2); + EXPECT_EQ(results.return_tensors[0].first->name(), "import/new_input"); + EXPECT_EQ(results.return_tensors[0].second, 0); + EXPECT_EQ(results.return_tensors[1].first->name(), "input"); + EXPECT_EQ(results.return_tensors[1].second, 0); // Test returning node remapped to source node opts.prefix.clear(); opts.input_map.clear(); opts.return_tensors.clear(); - return_tensors.clear(); + results = ImportGraphDefResults(); opts.input_map[{"new_input", 0}] = {"_SOURCE", 0}; opts.return_tensors.push_back({"new_input", 0}); ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner, - &return_tensors); + &results); EXPECT_TRUE(HasNode("new_input")); - ASSERT_EQ(return_tensors.size(), 1); - EXPECT_EQ(return_tensors[0].first->name(), "_SOURCE"); - EXPECT_EQ(return_tensors[0].second, 0); + ASSERT_EQ(results.return_tensors.size(), 1); + EXPECT_EQ(results.return_tensors[0].first->name(), "_SOURCE"); + EXPECT_EQ(results.return_tensors[0].second, 0); } TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensorsErrors) { - // Passing in return_tensors with empty opts.return_tensors is OK + // Null results with non-empty opts.return_tensors ImportGraphDefOptions opts; - std::vector> return_tensors; - ExpectOK("node { name: 'input' op: 'TestInput' }", opts, nullptr, - &return_tensors); - - // Null return_tensors with non-empty opts.return_tensors opts.return_tensors.push_back({"new_input", 0}); ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, - {"return_tensors argument to ImportGraphDef() must be non-null " - "if opts.return_tensors is non-empty"}); + {"results argument to ImportGraphDef() must be non-null if " + "opts.return_tensors is non-empty"}); - // Non-empty return_tensors - return_tensors.push_back({nullptr, 0}); - ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, - {"return_tensors argument to ImportGraphDef() should be empty " - "(has size 1)"}, - nullptr, &return_tensors); + // Non-empty results.return_tensors + ImportGraphDefResults results; + results.return_tensors.push_back({nullptr, 0}); + ExpectError( + "node { name: 'new_input' op: 'TestInput' }", opts, + {"All fields in results argument to ImportGraphDef() must be empty."}, + nullptr, &results); // Requesting tensor that isn't in graph def - return_tensors.clear(); + results.return_tensors.clear(); ExpectError("node { name: 'W1' op: 'TestParams' }", opts, - {"Requested return node 'new_input' not found in graph def"}, - nullptr, &return_tensors); + {"Requested return tensor 'new_input:0' not found in graph def"}, + nullptr, &results); // Requesting invalid node index opts.return_tensors.clear(); @@ -1655,7 +1646,89 @@ TEST_F(GraphConstructorTest, ImportGraphDef_ReturnTensorsErrors) { ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, {"Invalid return output 2 of node 'new_input', which has 2 " "output(s)"}, - nullptr, &return_tensors); + nullptr, &results); +} + +TEST_F(GraphConstructorTest, ImportGraphDef_ReturnNodes) { + ShapeRefiner refiner(TF_GRAPH_DEF_VERSION, graph_.op_registry()); + + ImportGraphDefOptions opts; + opts.return_nodes.push_back("input"); + opts.return_nodes.push_back("t1"); + ImportGraphDefResults results; + ExpectOK( + "node { name: 'input' op: 'TestInput' }" + "node { name: 'input2' op: 'TestInput' }" + "node { name: 't1' op: 'TestMul' input: ['input:0', 'input2:1'] }", + opts, &refiner, &results); + + // Sanity checks + EXPECT_TRUE(HasNode("input")); + EXPECT_TRUE(HasNode("input2")); + EXPECT_TRUE(HasNode("t1")); + EXPECT_TRUE(HasEdge("input", 0, "t1", 0)); + EXPECT_TRUE(HasEdge("input2", 1, "t1", 1)); + + // Check return tensors + ASSERT_EQ(results.return_nodes.size(), 2); + EXPECT_EQ(results.return_tensors.size(), 0); + EXPECT_EQ(results.unused_input_map_keys.size(), 0); + EXPECT_EQ(results.return_nodes[0]->name(), "input"); + EXPECT_EQ(results.return_nodes[1]->name(), "t1"); + + // Test using prefix + opts = ImportGraphDefOptions(); + results = ImportGraphDefResults(); + opts.prefix = "import"; + opts.return_nodes.push_back("input"); + ExpectOK("node { name: 'input' op: 'TestInput' }", opts, &refiner, &results); + + EXPECT_TRUE(HasNode("import/input")); + + ASSERT_EQ(results.return_nodes.size(), 1); + EXPECT_EQ(results.return_nodes[0]->name(), "import/input"); + + // Test that input_map has no effect + opts = ImportGraphDefOptions(); + results = ImportGraphDefResults(); + opts.input_map[{"new_input", 0}] = {"input", 0}; + opts.return_nodes.push_back("new_input"); + ExpectOK("node { name: 'new_input' op: 'TestInput' }", opts, &refiner, + &results); + + EXPECT_TRUE(HasNode("new_input")); + + ASSERT_EQ(results.return_nodes.size(), 1); + EXPECT_EQ(results.return_nodes[0]->name(), "new_input"); +} + +TEST_F(GraphConstructorTest, ImportGraphDef_ReturnNodesErrors) { + // Null results with non-empty opts.return_nodes + ImportGraphDefOptions opts; + opts.return_nodes.push_back("new_input"); + ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, + {"results argument to ImportGraphDef() must be non-null if " + "opts.return_nodes is non-empty"}); + + // Non-empty results.return_nodes + ImportGraphDefResults results; + results.return_nodes.push_back(nullptr); + ExpectError( + "node { name: 'new_input' op: 'TestInput' }", opts, + {"All fields in results argument to ImportGraphDef() must be empty."}, + nullptr, &results); + + // Requesting node that isn't in graph def + results.return_nodes.clear(); + ExpectError("node { name: 'W1' op: 'TestParams' }", opts, + {"Requested return node 'new_input' not found in graph def"}, + nullptr, &results); + + // Requesting return_nodes with skip_mapped_nodes not yet implemented + opts.skip_mapped_nodes = true; + ExpectError("node { name: 'new_input' op: 'TestInput' }", opts, + {"Requesting return_nodes with skip_mapped_nodes set is not " + "currently supported"}); } TEST_F(GraphConstructorTest, ImportGraphDef_WithCycle) { diff --git a/tensorflow/core/graph/testlib.cc b/tensorflow/core/graph/testlib.cc index be524387474e8863223ef0201fac8072fa5ad83e..172471e34bc5ce344a4a8db2d404b77b7406c99f 100644 --- a/tensorflow/core/graph/testlib.cc +++ b/tensorflow/core/graph/testlib.cc @@ -480,6 +480,24 @@ Node* Conv2D(Graph* g, Node* in0, Node* in1) { return ret; } +Node* Diag(Graph* g, Node* in, DataType type) { + Node* ret; + TF_CHECK_OK(NodeBuilder(g->NewName("n"), "Diag") + .Input(in) + .Attr("T", type) + .Finalize(g, &ret)); + return ret; +} + +Node* DiagPart(Graph* g, Node* in, DataType type) { + Node* ret; + TF_CHECK_OK(NodeBuilder(g->NewName("n"), "DiagPart") + .Input(in) + .Attr("T", type) + .Finalize(g, &ret)); + return ret; +} + void ToGraphDef(Graph* g, GraphDef* gdef) { g->ToGraphDef(gdef); } } // end namespace graph diff --git a/tensorflow/core/graph/testlib.h b/tensorflow/core/graph/testlib.h index a38809e6b4c04fbc98c27d3fdaa43dc43f37bf56..06597778bb204c83dae7699e1ffe0e2b196ac160 100644 --- a/tensorflow/core/graph/testlib.h +++ b/tensorflow/core/graph/testlib.h @@ -199,6 +199,12 @@ Node* BiasAdd(Graph* g, Node* value, Node* bias); // Add a Conv2D node in "g". Node* Conv2D(Graph* g, Node* in0, Node* in1); +// Add a Diag node in "g". +Node* Diag(Graph* g, Node* in, DataType type); + +// Add a DiagPart node in "g". +Node* DiagPart(Graph* g, Node* in, DataType type); + } // end namespace graph } // end namespace test } // end namespace tensorflow diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc index f62a21ace5935a562d53f8a2021308edad353fad..e9cb2ee09d52d5438c80d4601623c47eaf973a8c 100644 --- a/tensorflow/core/grappler/costs/graph_properties.cc +++ b/tensorflow/core/grappler/costs/graph_properties.cc @@ -195,9 +195,12 @@ Status GraphProperties::RelaxEnqueueShapesAndMergeTypes( Status GraphProperties::InferStatically() { Graph graph(OpRegistry::Global()); + FunctionLibraryDefinition function_library(graph.op_registry(), + item_.graph.library()); ShapeRefiner shape_refiner(graph.versions(), graph.op_registry()); shape_refiner.set_require_shape_inference_fns(false); shape_refiner.set_disable_constant_propagation(true); + shape_refiner.set_function_library_for_shape_inference(&function_library); ImportGraphDefOptions options; Status s = ImportGraphDef(options, item_.graph, &graph, &shape_refiner); TF_RETURN_IF_ERROR(s); diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc index 975ec31b146cba5b959056280bbf6ccf155b4eb6..134db5ec5a9ee11949c4ee6f869839e842089740 100644 --- a/tensorflow/core/grappler/costs/graph_properties_test.cc +++ b/tensorflow/core/grappler/costs/graph_properties_test.cc @@ -703,6 +703,36 @@ TEST_F(GraphPropertiesTest, InferRestoreOpShape_WithTwoNodesShareSameOutput) { EXPECT_EQ("float: [128,256]", PropToString(prop)); } +TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) { + // Test graph produced in python using: + /* + @function.Defun(*[tf.float32] * 2, noinline=True) + def MyAdd(x, y): + return tf.add(x,y) + + with tf.Graph().as_default(): + x = tf.constant(2.0, shape=[1, 2], dtype=tf.float32) + y = tf.constant(2.0, shape=[1, 2], dtype=tf.float32) + z = MyAdd(x, y) + z = MyAdd(x, z) + */ + // Check that the shape of the second MyAdd node propagates + // correctly. + GrapplerItem item; + string filename = io::JoinPath(testing::TensorFlowSrcRoot(), kTestDataPath, + "simple_function.pbtxt"); + TF_CHECK_OK(ReadGraphDefFromFile(filename, &item.graph)); + GraphProperties properties(item); + TF_CHECK_OK(properties.InferStatically()); + const auto props = properties.GetOutputProperties("MyAdd_55e046a8_1"); + const OpInfo::TensorProperties& prop = props[0]; + EXPECT_EQ(DT_FLOAT, prop.dtype()); + EXPECT_FALSE(prop.shape().unknown_rank()); + EXPECT_EQ(2, prop.shape().dim_size()); + EXPECT_EQ(1, prop.shape().dim(0).size()); + EXPECT_EQ(2, prop.shape().dim(1).size()); +} + } // namespace } // namespace grappler } // namespace tensorflow diff --git a/tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt b/tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..86b67f2049abb59ebd81ac8a71418ef25055bd21 --- /dev/null +++ b/tensorflow/core/grappler/costs/graph_properties_testdata/simple_function.pbtxt @@ -0,0 +1,111 @@ +node { + name: "Const" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 2 + } + } + float_val: 2.0 + } + } + } +} +node { + name: "Const_1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_FLOAT + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 2 + } + } + float_val: 2.0 + } + } + } +} +node { + name: "MyAdd_55e046a8" + op: "MyAdd_55e046a8" + input: "Const" + input: "Const_1" +} +node { + name: "MyAdd_55e046a8_1" + op: "MyAdd_55e046a8" + input: "Const" + input: "MyAdd_55e046a8" +} +library { + function { + signature { + name: "MyAdd_55e046a8" + input_arg { + name: "x" + type: DT_FLOAT + } + input_arg { + name: "y" + type: DT_FLOAT + } + output_arg { + name: "Add" + type: DT_FLOAT + } + } + node_def { + name: "Add" + op: "Add" + input: "x" + input: "y" + attr { + key: "T" + value { + type: DT_FLOAT + } + } + } + ret { + key: "Add" + value: "Add:z:0" + } + attr { + key: "_noinline" + value { + b: true + } + } + } +} +versions { + producer: 24 + min_consumer: 12 +} diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc index 54d60cd7aa41354267e23d65e6540d070a4937d1..3f6183b6f1ecb92dcc99abccacda74ceaf72cce0 100644 --- a/tensorflow/core/grappler/grappler_item_builder.cc +++ b/tensorflow/core/grappler/grappler_item_builder.cc @@ -450,12 +450,16 @@ std::unique_ptr GrapplerItemFromMetaGraphDef( } // Optimize the graph (function inlining, l1 optimizations, etc). + VLOG(1) << "Number of nodes in graph before OptimizeGraph: " + << new_item->graph.node_size(); Status optimize_status = OptimizeGraph(new_item->graph, &new_item->graph, cfg); if (!optimize_status.ok()) { LOG(ERROR) << "Graph preprocessing failed: " << optimize_status; return nullptr; } + VLOG(1) << "Number of nodes in graph after OptimizeGraph: " + << new_item->graph.node_size(); if (cfg.prune_graph) { VLOG(1) << "Pruning graph..."; @@ -464,7 +468,8 @@ std::unique_ptr GrapplerItemFromMetaGraphDef( LOG(ERROR) << "Pruning failed: " << status.error_message(); return nullptr; } - VLOG(1) << "Pruning ran succesfully."; + VLOG(1) << "Number of nodes in graph after pruning: " + << new_item->graph.node_size(); } // Validate feed, fetch and init nodes diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc index 8ef3383aa3eada00034c311f1399258107fdc53b..78b55237d1e665a296e945dafe0454afe722632e 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc @@ -29,6 +29,186 @@ limitations under the License. namespace tensorflow { namespace grappler { +namespace { + +static bool IsInvolution(const NodeDef& node) { + const std::unordered_set involution_ops = {"Conj", "Reciprocal", + "Neg", "LogicalNot"}; + return involution_ops.count(node.op()) > 0; +} + +bool AreInversePermutations(gtl::ArraySlice a, + gtl::ArraySlice b) { + if (a.size() != b.size()) { + return false; + } + for (int i = 0; i < a.size(); ++i) { + if (a[b[i]] != i) { + return false; + } + } + return true; +} + +// Extract int32 values from a Const op to `int32_values`. Returns true if +// succeeds. +bool Int32ValuesFromNode(const NodeDef& node, std::vector* int32_values) { + if (node.op() != "Const") { + return false; + } + + if (node.attr().at("dtype").type() != DT_INT32) { + return false; + } + + // TensorProto represents the content of the tensor in either _val or + // tensor_content. + const TensorProto& tensor = node.attr().at("value").tensor(); + if (tensor.int_val_size() > 0 && tensor.has_tensor_shape()) { + // When tensor_shape is set, theoretically the representation of the data + // could be compressed. So, before copying int_val to the returned vector, + // make sure no compression happens. + const TensorShapeProto& shape = tensor.tensor_shape(); + if (shape.dim_size() == 1 && shape.dim(0).size() == tensor.int_val_size()) { + int32_values->insert(int32_values->end(), tensor.int_val().begin(), + tensor.int_val().end()); + } + return true; + } + + const auto tensor_content_size = tensor.tensor_content().size(); + if (tensor_content_size > 0) { + CHECK_EQ(0, tensor_content_size % sizeof(int32)) + << "tensor_content_size (" << tensor_content_size + << ") is not a multiple of " << sizeof(int32); + int32_values->resize(tensor_content_size / sizeof(int32)); + port::CopyToArray(tensor.tensor_content(), + reinterpret_cast(int32_values->data())); + return true; + } + + return false; +} + +bool SimplyReordersData(const NodeDef& node) { + return node.op() == "Transpose"; +} + +// Returns the data type in attribute `attr_name` of `node`. If that attribute +// doesn't exist, returns DT_INVALID. +DataType GetDataTypeFromAttr(const NodeDef& node, const string& attr_name) { + if (!node.attr().count(attr_name)) { + return DT_INVALID; + } + const auto& attr = node.attr().at(attr_name); + if (attr.value_case() != AttrValue::kType) { + return DT_INVALID; + } + return attr.type(); +} + +bool IsCommutative(const OpDef& op, const NodeDef& input1) { + if (op.name() == "Add") { + // Workaround for "Add" not being marked is_commutative and is_aggregate. + // (See cl/173915048). + const auto type = GetDataTypeFromAttr(input1, "T"); + return type != DT_INVALID && type != DT_STRING; + } + return op.is_commutative(); +} + +void SetDataTypeToAttr(DataType dtype, const string& attr_name, NodeDef* node) { + (*node->mutable_attr())[attr_name].set_type(dtype); +} + +string SourceDataTypeAttrName(const NodeDef& node) { + if (node.op() == "Bitcast") { + return "T"; + } else if (node.op() == "Cast") { + return "SrcT"; + } else { + LOG(FATAL) << "SourceDataTypeAttrName not implemented for op " << node.op(); + } +} + +string DestinationDataTypeAttrName(const NodeDef& node) { + if (node.op() == "Bitcast") { + return "type"; + } else if (node.op() == "Cast") { + return "DstT"; + } else { + LOG(FATAL) << "DestinationDataTypeAttrName not implemented for op " + << node.op(); + } +} + +DataType GetSourceDataType(const NodeDef& node) { + return GetDataTypeFromAttr(node, SourceDataTypeAttrName(node)); +} + +DataType GetDestinationDataType(const NodeDef& node) { + return GetDataTypeFromAttr(node, DestinationDataTypeAttrName(node)); +} + +void SetSourceDataType(DataType dtype, NodeDef* node) { + SetDataTypeToAttr(dtype, SourceDataTypeAttrName(*node), node); +} + +bool IsNumberType(DataType dtype) { + DataTypeVector number_types = NumberTypes(); + return std::find(number_types.begin(), number_types.end(), dtype) != + number_types.end(); +} + +const char kOutputShapesAttr[] = "_output_shapes"; + +// Returns whether `reshape` is an identity op. The tensor that `reshape` +// reshapes is the `output_pos`-th output of node `input`. +bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input, + const int output_pos) { + if (!reshape.attr().count(kOutputShapesAttr) || + !input.attr().count(kOutputShapesAttr)) { + return false; + } + + PartialTensorShape src_shape( + input.attr().at(kOutputShapesAttr).list().shape(output_pos)); + PartialTensorShape dst_shape( + reshape.attr().at(kOutputShapesAttr).list().shape(0)); + if (src_shape.unknown_rank() || dst_shape.unknown_rank()) { + return false; + } + + if (!dst_shape.IsCompatibleWith(src_shape)) { + return false; + } + + // Returns false when src_shape or dst_shape has >=2 dimensions with unknown + // sizes. + auto num_unknown_dim_sizes = [](const PartialTensorShape& partial_shape) { + auto dim_sizes = partial_shape.dim_sizes(); + return std::count(dim_sizes.begin(), dim_sizes.end(), -1); + }; + int src_num_unknown_dim_sizes = num_unknown_dim_sizes(src_shape); + int dst_num_unknown_dim_sizes = num_unknown_dim_sizes(dst_shape); + if (src_num_unknown_dim_sizes > 1 || dst_num_unknown_dim_sizes > 1) { + return false; + } + + // Now, src_shape and dst_shape have at most one dimension with unknown + // sizes, and are compatible. Therefore, the reshape is a no-op when + // + // 1. at least one of them is fully-defined, or + // 2. both are partially defined and the -1 appears on the same dimension, + // i.e., IsIdenticalTo returns true. + if (src_num_unknown_dim_sizes == 1 && dst_num_unknown_dim_sizes == 1) { + return dst_shape.IsIdenticalTo(src_shape); + } + + return true; +} + +} // namespace class UniqueNodes { public: @@ -86,7 +266,7 @@ bool UniqueNodes::SameNode(const NodeDef& node1, const NodeDef& node2) const { // Compare inputs. const OpDef* op_def = nullptr; Status status = OpRegistry::Global()->LookUpOpDef(node1.op(), &op_def); - const bool is_commutative = status.ok() && op_def->is_commutative(); + const bool is_commutative = status.ok() && IsCommutative(*op_def, node1); if (is_commutative) { std::vector inputs1(node1.input().begin(), node1.input().end()); std::vector inputs2(node2.input().begin(), node2.input().end()); @@ -102,7 +282,6 @@ bool UniqueNodes::SameNode(const NodeDef& node1, const NodeDef& node2) const { if (IsControlInput(node1.input(index))) { ctrl_inputs1.push_back(node1.input(index)); ctrl_inputs2.push_back(node2.input(index)); - } else { regular_inputs1.push_back(node1.input(index)); regular_inputs2.push_back(node2.input(index)); @@ -218,177 +397,23 @@ void ArithmeticOptimizer::DedupComputations(GraphDef* optimized_graph) const { } } -static bool AreInversePermutations(gtl::ArraySlice a, - gtl::ArraySlice b) { - if (a.size() != b.size()) { - return false; - } - for (int i = 0; i < a.size(); ++i) { - if (a[b[i]] != i) { - return false; - } - } - return true; -} - -// Extract int32 values from a Const op to `int32_values`. Returns true if -// succeeds. -static bool Int32ValuesFromNode(const NodeDef& node, - std::vector* int32_values) { - if (node.op() != "Const") { - return false; - } - - if (node.attr().at("dtype").type() != DT_INT32) { - return false; - } - - // TensorProto represents the content of the tensor in either _val or - // tensor_content. - const TensorProto& tensor = node.attr().at("value").tensor(); - if (tensor.int_val_size() > 0 && tensor.has_tensor_shape()) { - // When tensor_shape is set, theoretically the representation of the data - // could be compressed. So, before copying int_val to the returned vector, - // make sure no compression happens. - const TensorShapeProto& shape = tensor.tensor_shape(); - if (shape.dim_size() == 1 && shape.dim(0).size() == tensor.int_val_size()) { - int32_values->insert(int32_values->end(), tensor.int_val().begin(), - tensor.int_val().end()); - } - return true; - } - - const auto tensor_content_size = tensor.tensor_content().size(); - if (tensor_content_size > 0) { - CHECK_EQ(0, tensor_content_size % sizeof(int32)) - << "tensor_content_size (" << tensor_content_size - << ") is not a multiple of " << sizeof(int32); - int32_values->resize(tensor_content_size / sizeof(int32)); - port::CopyToArray(tensor.tensor_content(), - reinterpret_cast(int32_values->data())); - return true; - } - - return false; -} - -static bool SimplyReordersData(const NodeDef& node) { - return node.op() == "Transpose"; -} - -// Returns the data type in attribute `attr_name` of `node`. If that attribute -// doesn't exist, returns DT_INVALID. -static DataType GetDataTypeFromAttr(const NodeDef& node, - const string& attr_name) { - if (!node.attr().count(attr_name)) { - return DT_INVALID; - } - const auto& attr = node.attr().at(attr_name); - if (attr.value_case() != AttrValue::kType) { - return DT_INVALID; - } - return attr.type(); -} - -static void SetDataTypeToAttr(DataType dtype, const string& attr_name, - NodeDef* node) { - (*node->mutable_attr())[attr_name].set_type(dtype); -} - -static string SourceDataTypeAttrName(const NodeDef& node) { - if (node.op() == "Bitcast") { - return "T"; - } else if (node.op() == "Cast") { - return "SrcT"; - } else { - LOG(FATAL) << "SourceDataTypeAttrName not implemented for op " << node.op(); - } -} - -static string DestinationDataTypeAttrName(const NodeDef& node) { - if (node.op() == "Bitcast") { - return "type"; - } else if (node.op() == "Cast") { - return "DstT"; - } else { - LOG(FATAL) << "DestinationDataTypeAttrName not implemented for op " - << node.op(); - } -} - -static DataType GetSourceDataType(const NodeDef& node) { - return GetDataTypeFromAttr(node, SourceDataTypeAttrName(node)); -} - -static DataType GetDestinationDataType(const NodeDef& node) { - return GetDataTypeFromAttr(node, DestinationDataTypeAttrName(node)); -} - -static void SetSourceDataType(DataType dtype, NodeDef* node) { - SetDataTypeToAttr(dtype, SourceDataTypeAttrName(*node), node); -} - -static bool IsNumberType(DataType dtype) { - DataTypeVector number_types = NumberTypes(); - return std::find(number_types.begin(), number_types.end(), dtype) != - number_types.end(); -} - -const char kOutputShapesAttr[] = "_output_shapes"; - -// Returns whether `reshape` is an identity op. The tensor that `reshape` -// reshapes is the `output_pos`-th output of node `input`. -static bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input, - const int output_pos) { - if (!reshape.attr().count(kOutputShapesAttr) || - !input.attr().count(kOutputShapesAttr)) { - return false; - } - - PartialTensorShape src_shape( - input.attr().at(kOutputShapesAttr).list().shape(output_pos)); - PartialTensorShape dst_shape( - reshape.attr().at(kOutputShapesAttr).list().shape(0)); - if (src_shape.unknown_rank() || dst_shape.unknown_rank()) { - return false; - } - - if (!dst_shape.IsCompatibleWith(src_shape)) { - return false; - } - - // Returns false when src_shape or dst_shape has >=2 dimensions with unknown - // sizes. - auto num_unknown_dim_sizes = [](const PartialTensorShape& partial_shape) { - auto dim_sizes = partial_shape.dim_sizes(); - return std::count(dim_sizes.begin(), dim_sizes.end(), -1); - }; - int src_num_unknown_dim_sizes = num_unknown_dim_sizes(src_shape); - int dst_num_unknown_dim_sizes = num_unknown_dim_sizes(dst_shape); - if (src_num_unknown_dim_sizes > 1 || dst_num_unknown_dim_sizes > 1) { - return false; - } - - // Now, src_shape and dst_shape have at most one dimension with unknown - // sizes, and are compatible. Therefore, the reshape is a no-op when - // - // 1. at least one of them is fully-defined, or - // 2. both are partially defined and the -1 appears on the same dimension, - // i.e., IsIdenticalTo returns true. - if (src_num_unknown_dim_sizes == 1 && dst_num_unknown_dim_sizes == 1) { - return dst_shape.IsIdenticalTo(src_shape); - } - - return true; -} - string ArithmeticOptimizer::TrySimplifyAndReplaceUses( const NodeDef* node, GraphDef* graph_def, NodeMap* node_map, std::vector* new_nodes) const { + // Remove involutions applied twice. + if (IsInvolution(*node)) { + // An involution is a function f(x) that is its own inverse, + // i.e. f(f(x)) = x. + const NodeDef* input = node_map->GetNode(node->input(0)); + if (input->op() == node->op()) { + return input->input(0); + } + } + // Remove inverse transposes. - if (node->op() == "Transpose") { + if (node->op() == "Transpose" || node->op() == "ConjugateTranspose") { const NodeDef* input = node_map->GetNode(node->input(0)); - if (input->op() == "Transpose") { + if (input->op() == node->op()) { const NodeDef* node_perm = node_map->GetNode(node->input(1)); const NodeDef* input_perm = node_map->GetNode(input->input(1)); std::vector node_perm_values; diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc index a4de838a65f328de0d6d965d74b1ff496fdfa782..61c8b82ea0fe70ce7d1463a646024546affed501 100644 --- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc @@ -77,6 +77,60 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) { EXPECT_EQ("c1", new_add.input(1)); } +TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output c1 = ops::Const(s.WithOpName("c1"), {1.0f, 2.0f}, {1, 2}); + Output c2 = ops::Const(s.WithOpName("c2"), {3.0f, 4.0f}, {1, 2}); + Output add1 = ops::Add(s.WithOpName("add1"), c1, c2); + Output add2 = ops::Add(s.WithOpName("add2"), c2, c1); + Output add3 = ops::Add(s.WithOpName("add3"), add1, add2); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ArithmeticOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + EXPECT_EQ(4, output.node_size()); + const NodeDef& new_c1 = output.node(0); + EXPECT_EQ("c1", new_c1.name()); + const NodeDef& new_c2 = output.node(1); + EXPECT_EQ("c2", new_c2.name()); + const NodeDef& new_add1 = output.node(2); + EXPECT_EQ("add1", new_add1.name()); + EXPECT_EQ(2, new_add1.input_size()); + EXPECT_EQ("c1", new_add1.input(0)); + EXPECT_EQ("c2", new_add1.input(1)); + const NodeDef& new_add3 = output.node(3); + EXPECT_EQ("add3", new_add3.name()); + EXPECT_EQ(2, new_add3.input_size()); + EXPECT_EQ("add1", new_add3.input(0)); + EXPECT_EQ("add1", new_add3.input(1)); +} + +TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsReal) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output c = ops::Const(s.WithOpName("c"), {1.0f, 2.0f}, {1, 2}); + Output neg1 = ops::Neg(s.WithOpName("neg1"), c); + Output neg2 = ops::Neg(s.WithOpName("neg2"), neg1); + Output recip1 = ops::Reciprocal(s.WithOpName("recip1"), neg2); + Output recip2 = ops::Reciprocal(s.WithOpName("recip2"), recip1); + Output id = ops::Identity(s.WithOpName("id"), recip2); + GrapplerItem item; + TF_CHECK_OK(s.ToGraphDef(&item.graph)); + + ArithmeticOptimizer optimizer; + GraphDef output; + Status status = optimizer.Optimize(nullptr, item, &output); + TF_EXPECT_OK(status); + + EXPECT_EQ(6, output.node_size()); + EXPECT_EQ("c", output.node(1).input(0)); + EXPECT_EQ("c", output.node(3).input(0)); + EXPECT_EQ("c", output.node(5).input(0)); +} + TEST_F(ArithmeticOptimizerTest, IdentityReshape) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output inputs = diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc index 11cab8099ae65727ed69cc8c78d9dfa865a5383d..b364446ad76bcc068ca4622067b92219e217c689 100644 --- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc @@ -54,6 +54,7 @@ std::set GetOpsFormatSupported() { "BiasAddGrad", "FusedBatchNorm", "FusedBatchNormGrad", + "FusedConv2DBiasActivation", "MaxPool", "MaxPoolGrad"}; return ops_format_supported; diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 2c025713465dfadccd582d91a7bc8b6b7677093a..1cb7c97be4f9fa86de499641f6c8165049625cde 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1098,7 +1098,7 @@ tf_kernel_library( visibility = [":friends"], deps = [ ":bounds_check", - "//tensorflow/core:framework_lite", + "//tensorflow/core:framework", "//third_party/eigen3", ], ) @@ -1691,6 +1691,21 @@ tf_cc_tests( ], ) +tf_kernel_library( + name = "eye_functor", + hdrs = ["eye_functor.h"], + gpu_srcs = [ + "eye_functor_gpu.cu.cc", + "eye_functor.h", + ], + visibility = [":friends"], + deps = [ + "//tensorflow/core:framework", + "//third_party/eigen3", + ], + alwayslink = 0, +) + cc_library( name = "fifo_queue", srcs = ["fifo_queue.cc"], @@ -2255,10 +2270,6 @@ tf_kernel_library( name = "cuda_solvers", srcs = ["cuda_solvers.cc"], hdrs = ["cuda_solvers.h"], - gpu_srcs = [ - "cuda_solvers.h", - "cuda_solvers_gpu.cu.cc", - ], # @local_config_cuda//cuda:cusolver, //third_party/eigen3:blas, # and //third_party/libf2c all contain various parts of BLAS, LAPACK, # and f2c helper functions in global namespace. Tell the compiler to @@ -2328,7 +2339,7 @@ tf_kernel_library( tf_kernel_library( name = "matrix_inverse_op", prefix = "matrix_inverse_op", - deps = LINALG_DEPS, + deps = LINALG_DEPS + if_cuda([":eye_functor"]), ) tf_kernel_library( @@ -2356,6 +2367,7 @@ tf_kernel_library( prefix = "qr_op", deps = LINALG_DEPS + if_cuda([ ":cwise_op", + ":eye_functor", ":matrix_band_part_op", ]), ) @@ -2499,6 +2511,7 @@ cc_library( ":cross_op", ":cwise_op", ":fft_ops", + ":histogram_op", ":matmul_op", ":population_count_op", ":reduction_ops", @@ -2635,6 +2648,24 @@ tf_kernel_library( deps = MATH_DEPS, ) +tf_cc_test( + name = "sequence_ops_test", + size = "small", + srcs = ["sequence_ops_test.cc"], + deps = [ + ":ops_testutil", + ":ops_util", + ":sequence_ops", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + tf_cuda_cc_test( name = "cast_op_test", size = "small", @@ -2893,6 +2924,24 @@ tf_cuda_cc_test( ], ) +tf_cuda_cc_test( + name = "diag_op_test", + size = "small", + srcs = ["diag_op_test.cc"], + deps = [ + ":diag_op", + ":ops_testutil", + ":ops_util", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + # conv_grad_ops currently has to be built with conv_ops*. # TODO(josh11b, zhengxq): put these a separate libraries in ":nn" below once # conv_ops_gpu.h has be separated into its own library. @@ -2993,6 +3042,7 @@ cc_library( ":in_topk_op", ":l2loss_op", ":lrn_op", + ":nth_element_op", ":relu_op", ":softmax_op", ":softplus_op", @@ -3079,6 +3129,12 @@ tf_kernel_library( deps = NN_DEPS + if_cuda(["@cub_archive//:cub"]), ) +tf_kernel_library( + name = "nth_element_op", + prefix = "nth_element_op", + deps = NN_DEPS, +) + tf_kernel_library( name = "xent_op", prefix = "xent_op", @@ -3096,6 +3152,17 @@ tf_kernel_library( ], ) +tf_kernel_library( + name = "histogram_op", + prefix = "histogram_op", + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//third_party/eigen3", + ] + if_cuda(["@cub_archive//:cub"]), +) + tf_kernel_library( name = "l2loss_op", prefix = "l2loss_op", @@ -3763,7 +3830,7 @@ STATE_DEPS = [ tf_kernel_library( name = "count_up_to_op", prefix = "count_up_to_op", - deps = STATE_DEPS, + deps = STATE_DEPS + [":variable_ops"], ) tf_kernel_library( @@ -4512,6 +4579,7 @@ filegroup( "cwise_op_greater_equal.cc", "cwise_op_invert.cc", "cwise_op_isfinite.cc", + "cwise_op_left_shift.cc", "cwise_op_less.cc", "cwise_op_less_equal.cc", "cwise_op_log.cc", @@ -4525,6 +4593,7 @@ filegroup( "cwise_op_neg.cc", "cwise_op_pow.cc", "cwise_op_reciprocal.cc", + "cwise_op_right_shift.cc", "cwise_op_round.cc", "cwise_op_rsqrt.cc", "cwise_op_select.cc", @@ -4812,6 +4881,7 @@ tf_kernel_library( deps = [ ":concat_lib_hdrs", ":conv_ops", + ":cwise_op", ":eigen_helpers", ":image_resizer_state", ":ops_util", @@ -5671,7 +5741,6 @@ cc_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", - "//tensorflow/core/util/tensor_bundle", ], ) @@ -5855,8 +5924,8 @@ tf_kernel_library( ) tf_kernel_library( - name = "sloppy_interleave_dataset_op", - srcs = ["sloppy_interleave_dataset_op.cc"], + name = "parallel_interleave_dataset_op", + srcs = ["parallel_interleave_dataset_op.cc"], deps = [ ":captured_function", ":dataset", @@ -6060,6 +6129,7 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", ], ) @@ -6092,6 +6162,7 @@ tf_kernel_library( ":map_and_batch_dataset_op", ":map_dataset_op", ":padded_batch_dataset_op", + ":parallel_interleave_dataset_op", ":parallel_map_dataset_op", ":prefetch_dataset_op", ":range_dataset_op", @@ -6100,7 +6171,6 @@ tf_kernel_library( ":scan_dataset_op", ":shuffle_dataset_op", ":skip_dataset_op", - ":sloppy_interleave_dataset_op", ":sparse_tensor_slice_dataset_op", ":sql_dataset_ops", ":take_dataset_op", diff --git a/tensorflow/core/kernels/batch_dataset_op.cc b/tensorflow/core/kernels/batch_dataset_op.cc index 631840081f4a4d26becb7ede7db059384438073c..04a41451ea5720f32b588fe98fa4e2ccf31828f9 100644 --- a/tensorflow/core/kernels/batch_dataset_op.cc +++ b/tensorflow/core/kernels/batch_dataset_op.cc @@ -181,7 +181,6 @@ class BatchDatasetOp : public UnaryDatasetOpKernel { private: mutex mu_; - int64 i_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; diff --git a/tensorflow/core/kernels/batchtospace_op.cc b/tensorflow/core/kernels/batchtospace_op.cc index 99b5d3daaa4b9ad2b962ceef986063229a887e7f..c1c0d6d329206088acaa009b3ffe695661527e44 100644 --- a/tensorflow/core/kernels/batchtospace_op.cc +++ b/tensorflow/core/kernels/batchtospace_op.cc @@ -249,40 +249,34 @@ class BatchToSpaceOp : public OpKernel { Tensor block_shape_; }; -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tcrops") \ - .HostMemory("block_shape") \ - .HostMemory("crops"), \ - BatchToSpaceNDOp); \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("crops"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("crops"), \ + BatchToSpaceNDOp); \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("crops"), \ BatchToSpaceOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER); #undef REGISTER #if GOOGLE_CUDA -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tcrops") \ - .HostMemory("block_shape") \ - .HostMemory("crops"), \ - BatchToSpaceNDOp); \ - REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("crops"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpaceND") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("crops"), \ + BatchToSpaceNDOp); \ + REGISTER_KERNEL_BUILDER(Name("BatchToSpace") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("crops"), \ BatchToSpaceOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER); diff --git a/tensorflow/core/kernels/candidate_sampler_ops.cc b/tensorflow/core/kernels/candidate_sampler_ops.cc index 9e8b122801ab6f9dd386313c2234929a9e691129..e937c4f11ba34e16d319b7b4dec317e81b6b8b2c 100644 --- a/tensorflow/core/kernels/candidate_sampler_ops.cc +++ b/tensorflow/core/kernels/candidate_sampler_ops.cc @@ -44,9 +44,11 @@ class BaseCandidateSamplerOp : public OpKernel { OP_REQUIRES(context, true_classes.dims() == 2, errors::InvalidArgument("true_classes must be a matrix")); const int32 batch_size = true_classes.dim_size(0); - OP_REQUIRES(context, true_classes.dim_size(1) == num_true_, - errors::InvalidArgument("true_classes must have " - "num_true columns")); + OP_REQUIRES( + context, true_classes.dim_size(1) == num_true_, + errors::InvalidArgument("true_classes must have " + "num_true columns, expected: ", + true_classes.dim_size(1), " was: ", num_true_)); CHECK(sampler_) << "CandidateSamplerOp did not set sampler_"; if (unique_) { diff --git a/tensorflow/core/kernels/concatenate_dataset_op.cc b/tensorflow/core/kernels/concatenate_dataset_op.cc index a6d27852b59499d91687cdf64b05b0cee52fdd7a..711c234129f7ca52667ca49600c35e2c8005652c 100644 --- a/tensorflow/core/kernels/concatenate_dataset_op.cc +++ b/tensorflow/core/kernels/concatenate_dataset_op.cc @@ -36,15 +36,17 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel { " have different output_types %s and %s", (DataTypeVectorString(input->output_dtypes()), DataTypeVectorString(to_concatenate->output_dtypes())))); - *output = new Dataset(input, to_concatenate); + *output = new Dataset(ctx, input, to_concatenate); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - explicit Dataset(const DatasetBase* input, + explicit Dataset(OpKernelContext* ctx, const DatasetBase* input, const DatasetBase* to_concatenate) - : input_(input), to_concatenate_(to_concatenate) { + : GraphDatasetBase(ctx), + input_(input), + to_concatenate_(to_concatenate) { input_->Ref(); to_concatenate_->Ref(); @@ -76,6 +78,19 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel { string DebugString() override { return "ConcatenateDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + Node* input_graph = nullptr; + TF_RETURN_IF_ERROR(b->AddParentDataset(input_, &input_graph)); + Node* to_concatenate_graph = nullptr; + TF_RETURN_IF_ERROR( + b->AddParentDataset(to_concatenate_, &to_concatenate_graph)); + TF_RETURN_IF_ERROR( + b->AddDataset(this, {input_graph, to_concatenate_graph}, output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -105,6 +120,30 @@ class ConcatenateDatasetOp : public BinaryDatasetOpKernel { return Status::OK(); } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); + TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_)); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); + if (!TF_PREDICT_TRUE(i_ >= 0 && i_ <= 2)) + return errors::InvalidArgument("i_ must be in range [0, 2]."); + if (i_ == 1) { + input_impl_ = dataset()->to_concatenate_->MakeIterator( + strings::StrCat(prefix(), "[1]")); + } else if (i_ == 2) { + input_impl_.reset(); + } + TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_)); + return Status::OK(); + } + private: mutex mu_; int64 i_ GUARDED_BY(mu_); diff --git a/tensorflow/core/kernels/count_up_to_op.cc b/tensorflow/core/kernels/count_up_to_op.cc index 040c40d606001fe6dfc553eb1ae8943cb5750424..9da0015fa2d97e45a3f00e6c289dfa40ea9f67c9 100644 --- a/tensorflow/core/kernels/count_up_to_op.cc +++ b/tensorflow/core/kernels/count_up_to_op.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/variable_ops.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" @@ -54,10 +55,56 @@ class CountUpToOp : public OpKernel { T limit_; }; -#define REGISTER(TYPE) \ - REGISTER_KERNEL_BUILDER( \ - Name("CountUpTo").TypeConstraint("T").Device(DEVICE_CPU), \ - CountUpToOp) +template +class ResourceCountUpToOp : public OpKernel { + public: + explicit ResourceCountUpToOp(OpKernelConstruction* context) + : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("limit", &limit_)); + OP_REQUIRES_OK(context, context->GetAttr("T", &dtype_)); + } + + void Compute(OpKernelContext* context) override { + Var* variable = nullptr; + OP_REQUIRES_OK( + context, + LookupResource(context, HandleFromInput(context, 0), &variable)); + core::ScopedUnref s(variable); + mutex_lock l(*variable->mu()); + Tensor before_increment = *variable->tensor(); + OP_REQUIRES( + context, TensorShapeUtils::IsScalar(before_increment.shape()), + errors::InvalidArgument("input is not a scalar: ", + before_increment.shape().DebugString())); + if (before_increment.scalar()() >= limit_) { + context->SetStatus(errors::OutOfRange("Reached limit of ", limit_)); + return; + } + // Allocate new buffer + AllocatorAttributes attr; + attr.set_gpu_compatible(true); + attr.set_nic_compatible(true); + PersistentTensor unused; + Tensor* tmp; + OP_REQUIRES_OK(context, context->allocate_persistent( + dtype_, TensorShape({}), &unused, &tmp, attr)); + *variable->tensor() = *tmp; + tmp->scalar()() = before_increment.scalar()() + 1; + context->set_output(0, before_increment); + } + + private: + T limit_; + DataType dtype_; +}; + +#define REGISTER(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("CountUpTo").TypeConstraint("T").Device(DEVICE_CPU), \ + CountUpToOp) \ + REGISTER_KERNEL_BUILDER( \ + Name("ResourceCountUpTo").TypeConstraint("T").Device(DEVICE_CPU), \ + ResourceCountUpToOp) REGISTER(int32); REGISTER(int64); diff --git a/tensorflow/core/kernels/crop_and_resize_op_test.cc b/tensorflow/core/kernels/crop_and_resize_op_test.cc index 22c659b587be4b981948fba19e091b6ecbf34481..a35e1b0788dbc60d6609faf1dfb97d5e7e4f515b 100644 --- a/tensorflow/core/kernels/crop_and_resize_op_test.cc +++ b/tensorflow/core/kernels/crop_and_resize_op_test.cc @@ -61,8 +61,12 @@ class CropAndResizeOpTest : public OpsTestBase { REGISTER_TEST(float) REGISTER_TEST(double) -REGISTER_TEST(int8) REGISTER_TEST(uint8) +REGISTER_TEST(uint16) +REGISTER_TEST(int8) +REGISTER_TEST(int16) +REGISTER_TEST(int32) +REGISTER_TEST(int64) #undef REGISTER_TEST diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/kernels/cuda_solvers.h index 60c4a0bfb47626e084a888d08509a954e4a1ffe4..3c389a82ab4070d5fb1bf3a091a4c85a6309eda9 100644 --- a/tensorflow/core/kernels/cuda_solvers.h +++ b/tensorflow/core/kernels/cuda_solvers.h @@ -27,7 +27,6 @@ limitations under the License. #include "cuda/include/cusolverDn.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/stream_executor.h" @@ -408,37 +407,6 @@ class DeviceLapackInfo : public ScratchSpace { } }; -namespace functor { -// Helper functor to transpose and conjugate all matrices in a flattened batch. -template -struct AdjointBatchFunctor { - // We assume that the tensor sizes are correct. - void operator()(const Device& device, - typename TTypes::ConstTensor input, - typename TTypes::Tensor output); -}; - -// Helper functor to compute the product of diagonal elements in all matrices -// in a flattened batch. -template -struct DeterminantFromPivotedLUFunctor { - void operator()(const Device& device, - typename TTypes::ConstTensor lu_factor, - const int* pivots, typename TTypes::Tensor output, - int* info); -}; - -// Helper functor to set a batch of matrices to the identity. -// TODO(rmlarsen): Use this kernel to replace the horribly inefficient tf.eye -// op. -template -struct EyeFunctor { - void operator()(const Device& device, - typename TTypes::Tensor matrix_batch); -}; - -} // namespace functor - template ScratchSpace CudaSolver::GetScratchSpace(const TensorShape& shape, const string& debug_info, diff --git a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc b/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc deleted file mode 100644 index 79961c01caebd32b0af18e2bb3eba25a16c2e918..0000000000000000000000000000000000000000 --- a/tensorflow/core/kernels/cuda_solvers_gpu.cu.cc +++ /dev/null @@ -1,247 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#if GOOGLE_CUDA - -#define EIGEN_USE_GPU - -#include "tensorflow/core/kernels/cuda_solvers.h" - -#include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/util/cuda_kernel_helper.h" - -namespace tensorflow { -namespace functor { - -typedef Eigen::GpuDevice GPUDevice; - -// TODO(rmlarsen): Add a faster custom kernel similar to -// SwapDimension1And2InTensor3 in tensorflow/core/kernels/conv_ops_gpu_3.cu.cc -template -struct AdjointBatchFunctor { - void operator()(const GPUDevice& device, - typename TTypes::ConstTensor input, - typename TTypes::Tensor output) { - const Eigen::array perm({0, 2, 1}); - To32Bit(output).device(device) = To32Bit(input).shuffle(perm).conjugate(); - } -}; - -// Instantiate implementations for the 4 numeric types. -template struct AdjointBatchFunctor; -template struct AdjointBatchFunctor; -template struct AdjointBatchFunctor; -template struct AdjointBatchFunctor; - -namespace { - -// Hacks around missing support for complex arithmetic in nvcc. -template -__device__ inline Scalar Multiply(Scalar x, Scalar y) { - return x * y; -} - -template <> -__device__ inline cuComplex Multiply(cuComplex x, cuComplex y) { - return cuCmulf(x, y); -} - -template <> -__device__ inline cuDoubleComplex Multiply(cuDoubleComplex x, - cuDoubleComplex y) { - return cuCmul(x, y); -} - -template -__device__ inline Scalar Negate(Scalar x) { - return -x; -} - -template <> -__device__ inline cuComplex Negate(cuComplex x) { - return make_cuComplex(-cuCrealf(x), -cuCimagf(x)); -} - -template <> -__device__ inline cuDoubleComplex Negate(cuDoubleComplex x) { - return make_cuDoubleComplex(-cuCreal(x), -cuCimag(x)); -} - -template -__device__ inline bool IsFinite(Scalar x) { - return Eigen::numext::isfinite(x); -} - -template <> -__device__ inline bool IsFinite(cuComplex x) { - return Eigen::numext::isfinite(cuCrealf(x)) && - Eigen::numext::isfinite(cuCimagf(x)); -} - -template <> -__device__ inline bool IsFinite(cuDoubleComplex x) { - return Eigen::numext::isfinite(cuCreal(x)) && - Eigen::numext::isfinite(cuCimag(x)); -} - -template -struct Const { - template - __device__ static inline Scalar make_const(const RealScalar x) { - return Scalar(x); - } -}; - -template <> -struct Const { - template - __device__ static inline cuComplex make_const(const RealScalar x) { - return make_cuComplex(x, 0.0f); - } -}; - -template <> -struct Const { - template - __device__ static inline cuDoubleComplex make_const(const RealScalar x) { - return make_cuDoubleComplex(x, 0.0f); - } -}; - -} // namespace - -template -__global__ void DeterminantFromPivotedLUKernel(int nthreads, int n, - const Scalar* lu_factor, - const int* all_pivots, - Scalar* dst, int* info) { - const int matrix_size = n * n; - const int stride = n + 1; - // We only parallelize over batches here. Performance is not critical, - // since this cheap O(n) kernel always follows an O(n^3) LU factorization. - // The main purpose is to avoid having to copy the LU decomposition to - // host memory. - CUDA_1D_KERNEL_LOOP(o_idx, nthreads) { - // Compute the order of the permutation from the number of transpositions - // encoded in the pivot array, see: - // http://icl.cs.utk.edu/lapack-forum/viewtopic.php?f=2&t=340 - const int* pivots = all_pivots + o_idx * n; - int order = 0; - for (int i = 0; i < n - 1; ++i) { - // Notice: Internally, the cuBlas code uses Fortran convention (1-based) - // indexing so we expect pivots[i] == i + 1 for rows that were not moved. - order += pivots[i] != (i + 1); - } - - // Compute the product of the diagonal elements of U from the partially - // pivoted LU factorization. - // TODO(rmlarsen): This naive implementation (matching that in Eigen used - // for the CPU kernel) is pathetically unstable. Should we implement - // log-determinant instead (a different set of ops altogether) or something - // like the method used in the old LINPACK code: - // http://www.netlib.org/linpack/dgedi.f ? - int i_idx = matrix_size * o_idx; - Scalar prod = lu_factor[i_idx]; - for (int i = 1; i < n; ++i) { - i_idx += stride; - prod = Multiply(prod, lu_factor[i_idx]); - } - // Finally set the determinant to (-1)^order * prod(diag(U)). - dst[o_idx] = order % 2 ? Negate(prod) : prod; - - // We write a magic value into the info array if the result was infinite. - if (!IsFinite(prod)) { - info[o_idx] = kint32min; - } - } -} - -template -struct DeterminantFromPivotedLUFunctor { - void operator()(const GPUDevice& device, - typename TTypes::ConstTensor lu_factor, - const int* pivots, typename TTypes::Tensor output, - int* info) { - using CudaType = typename CUDAComplexT::type; - const int64 num_matrices = output.size(); - const int64 n = lu_factor.dimension(2); - const CudaType* lu_factor_ptr = - reinterpret_cast(lu_factor.data()); - CudaType* output_ptr = reinterpret_cast(output.data()); - CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device); - DeterminantFromPivotedLUKernel<<< - config.block_count, config.thread_per_block, 0, device.stream()>>>( - config.virtual_thread_count, n, lu_factor_ptr, pivots, output_ptr, - info); - } -}; - -template struct DeterminantFromPivotedLUFunctor; -template struct DeterminantFromPivotedLUFunctor; -template struct DeterminantFromPivotedLUFunctor; -template struct DeterminantFromPivotedLUFunctor; - -template -__global__ void EyeKernel(Cuda3DLaunchConfig config, int batch_size, int m, - int n, Scalar* matrix_batch_ptr) { - const int matrix_size = m * n; - const Scalar one = Const::make_const(1.0); - CUDA_AXIS_KERNEL_LOOP(batch, config.virtual_thread_count, x) { - if (batch >= batch_size) { - break; - } - CUDA_AXIS_KERNEL_LOOP(row, config.virtual_thread_count, y) { - if (row >= m) { - break; - } - const int row_start = batch * matrix_size + row * n; - CUDA_AXIS_KERNEL_LOOP(col, config.virtual_thread_count, z) { - if (col >= n) { - break; - } - matrix_batch_ptr[row_start + col] = row == col ? one : Scalar(); - } - } - } -} - -template -struct EyeFunctor { - void operator()(const GPUDevice& device, - typename TTypes::Tensor matrix_batch) { - using CudaType = typename CUDAComplexT::type; - const int batch_size = matrix_batch.dimension(0); - const int m = matrix_batch.dimension(1); - const int n = matrix_batch.dimension(2); - CudaType* matrix_batch_ptr = - reinterpret_cast(matrix_batch.data()); - Cuda3DLaunchConfig config = GetCuda3DLaunchConfig(batch_size, m, n, device, - EyeKernel, 0, 0); - EyeKernel<<>>(config, batch_size, m, n, matrix_batch_ptr); - } -}; - -template struct EyeFunctor; -template struct EyeFunctor; -template struct EyeFunctor; -template struct EyeFunctor; - -} // namespace functor -} // namespace tensorflow - -#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/cwise_op_add_1.cc b/tensorflow/core/kernels/cwise_op_add_1.cc index c0fe81ef553028c427bc145afb47641e291bf778..608a6dce3d223d522776c59a3a1b2ad0d0c14147 100644 --- a/tensorflow/core/kernels/cwise_op_add_1.cc +++ b/tensorflow/core/kernels/cwise_op_add_1.cc @@ -18,9 +18,12 @@ limitations under the License. namespace tensorflow { REGISTER5(BinaryOp, CPU, "Add", functor::add, float, Eigen::half, double, int32, int64); +REGISTER5(BinaryOp, CPU, "AddV2", functor::add, float, Eigen::half, double, + int32, int64); #if GOOGLE_CUDA REGISTER3(BinaryOp, GPU, "Add", functor::add, float, Eigen::half, double); +REGISTER3(BinaryOp, GPU, "AddV2", functor::add, float, Eigen::half, double); // A special GPU kernel for int32. // TODO(b/25387198): Also enable int32 in device memory. This kernel @@ -32,11 +35,21 @@ REGISTER_KERNEL_BUILDER(Name("Add") .HostMemory("z") .TypeConstraint("T"), BinaryOp>); +REGISTER_KERNEL_BUILDER(Name("AddV2") + .Device(DEVICE_GPU) + .HostMemory("x") + .HostMemory("y") + .HostMemory("z") + .TypeConstraint("T"), + BinaryOp>); #endif #if TENSORFLOW_USE_SYCL -#define REGISTER_KERNEL(type) REGISTER(BinaryOp, SYCL, "Add", functor::add, type); +#define REGISTER_KERNEL(type) \ + REGISTER(BinaryOp, SYCL, "Add", functor::add, type); \ + REEGISTER(BinaryOp, SYCL, "AddV2", functor::add, type); + TF_CALL_SYCL_NUMBER_TYPES(REGISTER_KERNEL); REGISTER_KERNEL_BUILDER(Name("Add") @@ -46,5 +59,12 @@ REGISTER_KERNEL_BUILDER(Name("Add") .HostMemory("z") .TypeConstraint("T"), BinaryOp>); +REGISTER_KERNEL_BUILDER(Name("AddV2") + .Device(DEVICE_SYCL) + .HostMemory("x") + .HostMemory("y") + .HostMemory("z") + .TypeConstraint("T"), + BinaryOp>); #endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_add_2.cc b/tensorflow/core/kernels/cwise_op_add_2.cc index 5dea00e95c7a653ac050bfde41910a861e5b8fd4..ac21ca06c929662271ad99b3756b8a22fc62a0cf 100644 --- a/tensorflow/core/kernels/cwise_op_add_2.cc +++ b/tensorflow/core/kernels/cwise_op_add_2.cc @@ -24,9 +24,15 @@ namespace tensorflow { REGISTER6(BinaryOp, CPU, "Add", functor::add, int8, int16, complex64, uint8, complex128, string); +// Notice: String is excluded to allow marking AddV2 is_commutative and +// is_aggregate. +REGISTER5(BinaryOp, CPU, "AddV2", functor::add, int8, int16, complex64, uint8, + complex128); #if GOOGLE_CUDA REGISTER4(BinaryOp, GPU, "Add", functor::add, uint8, int64, complex64, complex128); +REGISTER4(BinaryOp, GPU, "AddV2", functor::add, uint8, int64, complex64, + complex128); #endif // GOOGLE_CUDA #endif // !defined(__ANDROID_TYPES_SLIM__) diff --git a/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..740048795a988e30447e6bcba44374f16c0dcdb4 --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_gpu_left_shift.cu.cc @@ -0,0 +1,27 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h" + +namespace tensorflow { +namespace functor { +DEFINE_BINARY8(left_shift, int8, int16, int32, int64, uint8, uint16, uint32, + uint64); +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc b/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..bb6772772c8d3391792bea173ea3a1c55691bdee --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_gpu_right_shift.cu.cc @@ -0,0 +1,27 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#include "tensorflow/core/kernels/cwise_ops_gpu_common.cu.h" + +namespace tensorflow { +namespace functor { +DEFINE_BINARY8(right_shift, int8, int16, int32, int64, uint8, uint16, uint32, + uint64); +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/cwise_op_left_shift.cc b/tensorflow/core/kernels/cwise_op_left_shift.cc new file mode 100644 index 0000000000000000000000000000000000000000..ccb68139dec6ff5a7a13088cc42033c4c6c9d461 --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_left_shift.cc @@ -0,0 +1,44 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/kernels/cwise_ops_common.h" + +namespace tensorflow { +REGISTER8(BinaryOp, CPU, "LeftShift", functor::left_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); + +#if TENSORFLOW_USE_SYCL +#define REGISTER_SYCL_KERNEL(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("LeftShift").Device(DEVICE_SYCL).TypeConstraint("T"), \ + BinaryOp>); +REGISTER_SYCL_KERNEL(int8); +REGISTER_SYCL_KERNEL(int16); +REGISTER_SYCL_KERNEL(int32); +REGISTER_SYCL_KERNEL(int64); +REGISTER_SYCL_KERNEL(uint8); +REGISTER_SYCL_KERNEL(uint16); +REGISTER_SYCL_KERNEL(uint32); +REGISTER_SYCL_KERNEL(uint64); +#undef REGISTER_SYCL_KERNEL + +#endif // TENSORFLOW_USE_SYCL + +#if GOOGLE_CUDA +REGISTER8(BinaryOp, GPU, "LeftShift", functor::left_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); +#endif // GOOGLE_CUDA + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_op_right_shift.cc b/tensorflow/core/kernels/cwise_op_right_shift.cc new file mode 100644 index 0000000000000000000000000000000000000000..6dc6b97e35418990a231c26ec0acdc14db31d887 --- /dev/null +++ b/tensorflow/core/kernels/cwise_op_right_shift.cc @@ -0,0 +1,44 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/kernels/cwise_ops_common.h" + +namespace tensorflow { +REGISTER8(BinaryOp, CPU, "RightShift", functor::right_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); + +#if TENSORFLOW_USE_SYCL +#define REGISTER_SYCL_KERNEL(TYPE) \ + REGISTER_KERNEL_BUILDER( \ + Name("RightShift").Device(DEVICE_SYCL).TypeConstraint("T"), \ + BinaryOp>); +REGISTER_SYCL_KERNEL(int8); +REGISTER_SYCL_KERNEL(int16); +REGISTER_SYCL_KERNEL(int32); +REGISTER_SYCL_KERNEL(int64); +REGISTER_SYCL_KERNEL(uint8); +REGISTER_SYCL_KERNEL(uint16); +REGISTER_SYCL_KERNEL(uint32); +REGISTER_SYCL_KERNEL(uint64); +#undef REGISTER_SYCL_KERNEL + +#endif // TENSORFLOW_USE_SYCL + +#if GOOGLE_CUDA +REGISTER8(BinaryOp, GPU, "RightShift", functor::right_shift, int8, int16, int32, + int64, uint8, uint16, uint32, uint64); +#endif // GOOGLE_CUDA + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h index d935331904d44dba0ba49c0a0f54a4bdca68a391..89487419ee9132320613b88950aab138f34512f4 100644 --- a/tensorflow/core/kernels/cwise_ops.h +++ b/tensorflow/core/kernels/cwise_ops.h @@ -18,6 +18,8 @@ limitations under the License. #include #include +#include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/tensor_types.h" @@ -810,6 +812,50 @@ struct bitwise_or : base> {}; template struct bitwise_xor : base> {}; +template +struct left_shift_op { + EIGEN_EMPTY_STRUCT_CTOR(left_shift_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x, + const T& y) const { + // Avoids UB: don't shift by larger than the bitwidth of T, and + // performs left shifts as unsigned shifts. + T y_clamped = y; + if (y_clamped < 0) { + y_clamped = 0; + } else if (y_clamped > sizeof(T) * CHAR_BIT - 1) { + y_clamped = sizeof(T) * CHAR_BIT - 1; + } + using U = typename std::make_unsigned::type; + return static_cast(static_cast(x) << static_cast(y_clamped)); + } +}; + +template +struct right_shift_op { + EIGEN_EMPTY_STRUCT_CTOR(right_shift_op) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T operator()(const T& x, + const T& y) const { + // Avoids UB: don't shift by larger than the bitwidth of T. + T y_clamped = y; + if (y_clamped < 0) { + y_clamped = 0; + } else if (y_clamped > sizeof(T) * CHAR_BIT - 1) { + y_clamped = sizeof(T) * CHAR_BIT - 1; + } + // Technically right shifts of signed integers are not necessarily + // arithmetic shifts according to the C++ standard. However in practice most + // implementations are arithmetic shifts. If this proves to be a problem in + // practice, we may need to use an alternative implementation. + return x >> y_clamped; + } +}; + +template +struct left_shift : base> {}; + +template +struct right_shift : base> {}; + template struct make_complex_func { typedef std::complex result_type; diff --git a/tensorflow/core/kernels/cwise_ops_common.h b/tensorflow/core/kernels/cwise_ops_common.h index 9a05e1500f57c3203f29d9b667e9ebbb42894770..8295fa939ee1aabf78a7d7b7f4677d851b407573 100644 --- a/tensorflow/core/kernels/cwise_ops_common.h +++ b/tensorflow/core/kernels/cwise_ops_common.h @@ -305,6 +305,62 @@ struct BinaryFunctor { Assign(d, out, in.unaryExpr(Unary(scalar.data()))); } + void BCast(const CPUDevice& dev, + typename TTypes::Tensor out, + typename TTypes::ConstTensor in0, + typename Eigen::array bcast0, + typename TTypes::ConstTensor in1, + typename Eigen::array bcast1, + bool* error) { + typename Functor::func func; + if (AllOne(bcast0) && AllOne(bcast1)) { + Assign(dev, out, in0.binaryExpr(in1, func)); + } else if (AllOne(bcast0)) { + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, in0.binaryExpr(rhs, func)); + } else if (AllOne(bcast1)) { + auto lhs = in0.broadcast(bcast0); + Assign(dev, out, lhs.binaryExpr(in1, func)); + } else { + auto lhs = in0.broadcast(bcast0); + auto rhs = in1.broadcast(bcast1); + Assign(dev, out, lhs.binaryExpr(rhs, func)); + } + } +}; + +// Partial specialization of BinaryFunctor +// for functors with with no error checking. +template +struct BinaryFunctor { + enum { NDIMS = 2 }; + + void operator()(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tin_type in0, + typename Functor::tin_type in1, bool* error) { + Assign(d, out, in0.binaryExpr(in1, typename Functor::func())); + } + + void Left(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tscalar_type scalar, + typename Functor::tin_type in, bool* error) { + typedef typename Functor::out_type Tout; + typedef typename Functor::in_type Tin; + typedef typename Functor::func Binary; + typedef typename Eigen::internal::scalar_left Unary; + Assign(d, out, in.unaryExpr(Unary(scalar.data()))); + } + + void Right(const CPUDevice& d, typename Functor::tout_type out, + typename Functor::tin_type in, + typename Functor::tscalar_type scalar, bool* error) { + typedef typename Functor::out_type Tout; + typedef typename Functor::in_type Tin; + typedef typename Functor::func Binary; + typedef typename Eigen::internal::scalar_right Unary; + Assign(d, out, in.unaryExpr(Unary(scalar.data()))); + } + #if !defined(EIGEN_HAS_INDEX_LIST) inline Eigen::DSizes NByOne(int n) { return Eigen::DSizes(n, 1); @@ -334,8 +390,7 @@ struct BinaryFunctor { bool* error) { typedef typename Functor::in_type T; typename Functor::func func; - if ((NDIMS == 2) && Functor::use_bcast_optimization && - use_bcast_optimization::value) { + if (Functor::use_bcast_optimization && use_bcast_optimization::value) { // Optimize for speed by using Eigen::type2index and avoid // .broadcast() when we know its a no-op. // @@ -410,7 +465,7 @@ struct BinaryFunctor { } } - // Fallback path. Always work and probably slower. + // Fallback path. Always works and probably slower. auto lhs = in0.broadcast(bcast0); auto rhs = in1.broadcast(bcast1); Assign(dev, out, lhs.binaryExpr(rhs, func)); diff --git a/tensorflow/core/kernels/dataset.h b/tensorflow/core/kernels/dataset.h index 9486f478b624202aedc8ba88c31fc22d3fa50031..e0ffe268dd6630f60c375f0d6a7dc4ff62b06dc2 100644 --- a/tensorflow/core/kernels/dataset.h +++ b/tensorflow/core/kernels/dataset.h @@ -17,12 +17,14 @@ limitations under the License. #include +#include "tensorflow/core/common_runtime/graph_runner.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/resource_mgr.h" #include "tensorflow/core/framework/variant_encode_decode.h" #include "tensorflow/core/framework/variant_tensor_data.h" #include "tensorflow/core/graph/graph.h" +#include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/graph_def_builder.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" @@ -39,54 +41,25 @@ namespace tensorflow { class ResourceMgr; -class BundleReaderWrapper { +// Interface for reading values from a key-value store. +// Used for restoring iterator state. +class IteratorStateReader { public: - BundleReaderWrapper(BundleReader* bundle_reader) - : bundle_reader_(bundle_reader) {} + virtual Status ReadScalar(StringPiece key, int64* val) = 0; + virtual Status ReadScalar(StringPiece key, string* val) = 0; + virtual bool Contains(StringPiece key) = 0; - // Reads a scalar value. - template - Status ReadScalar(StringPiece key, T* val) { - Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); - TF_RETURN_IF_ERROR(Lookup(key, &val_t)); - *val = val_t.scalar()(); - return Status::OK(); - } - - bool Contains(StringPiece key) { return bundle_reader_->Contains(key); } - - private: - Status Lookup(StringPiece key, Tensor* val) { - return bundle_reader_->Lookup(key, val); - } - - BundleReader* bundle_reader_; + virtual ~IteratorStateReader() {} }; -class BundleWriterWrapper { +// Interface for writing values to a key-value store. +// Used for saving iterator state. +class IteratorStateWriter { public: - // Note: We intentionally do not provide a constructor that builds a - // BundleWriter from the checkpoint path because we want the caller to be - // in-charge of calling BundleWriter::Finish(). If we expose the Finish() - // method here it may be called pre-maturely by users of this object. - explicit BundleWriterWrapper(BundleWriter* bundle_writer) - : bundle_writer_(bundle_writer) {} - - // Writes a scalar value. - template - Status WriteScalar(StringPiece key, const T val) { - Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); - val_t.scalar()() = val; - TF_RETURN_IF_ERROR(Add(key, val_t)); - return Status::OK(); - } + virtual Status WriteScalar(StringPiece key, const int64 val) = 0; + virtual Status WriteScalar(StringPiece key, const string& val) = 0; - private: - Status Add(StringPiece key, const Tensor& val) { - return bundle_writer_->Add(key, val); - } - - BundleWriter* bundle_writer_; + virtual ~IteratorStateWriter() {} }; // Wrapper around GraphDefBuilder. Used to serialize Dataset graph. @@ -102,10 +75,7 @@ class GraphDefBuilderWrapper { Status AddScalar(const T& val, Node** output) { Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); val_t.scalar()() = val; - *output = - ops::SourceOp("Const", b_->opts() - .WithAttr("dtype", DataTypeToEnum::v()) - .WithAttr("value", val_t)); + AddTensorInternal(val_t, output); if (*output == nullptr) { return errors::Internal("AddScalar: Failed to build Const op."); } @@ -123,16 +93,25 @@ class GraphDefBuilderWrapper { for (int i = 0; i < val.size(); i++) { val_t.flat()(i) = val[i]; } - *output = - ops::SourceOp("Const", b_->opts() - .WithAttr("dtype", DataTypeToEnum::v()) - .WithAttr("value", val_t)); + AddTensorInternal(val_t, output); if (*output == nullptr) { return errors::Internal("AddVector: Failed to build Const op."); } return Status::OK(); } + // Adds a Const node with Tensor value to the Graph. + // `*output` contains a pointer to the output `Node`. It is guaranteed to be + // non-null if the method returns with an OK status. + // The returned Node pointer is owned by the backing Graph of GraphDefBuilder. + Status AddTensor(const Tensor& val, Node** output) { + AddTensorInternal(val, output); + if (*output == nullptr) { + return errors::Internal("AddTesor: Failed to build Const op."); + } + return Status::OK(); + } + // Adds a node corresponding to the `DatasetType` to the Graph. // Return value of `DatasetType::op_name()` is used as the op type for the // node. @@ -175,7 +154,46 @@ class GraphDefBuilderWrapper { return Status::OK(); } + // TODO(shivaniagrawal): Single method for AddDataset for + // NodeOut/ArrraySlice + template + Status AddDatasetWithInputAsList(const DatasetType* dataset, + gtl::ArraySlice input, + Node** output) { + const string& op_type_name = dataset->op_name(); + std::unique_ptr opts( + new GraphDefBuilder::Options(b_->opts())); + bool has_output_types_attr = HasAttr(op_type_name, "output_types"); + bool has_output_shapes_attr = HasAttr(op_type_name, "output_shapes"); + if (has_output_shapes_attr) { + opts.reset(new GraphDefBuilder::Options( + opts->WithAttr("output_shapes", dataset->output_shapes()))); + } + if (has_output_types_attr) { + opts.reset(new GraphDefBuilder::Options( + opts->WithAttr("output_types", dataset->output_dtypes()))); + } + if (opts->HaveError()) { + return errors::Internal("AddDataset: Error building Options."); + } + NodeBuilder node_builder(opts->GetNameForOp(op_type_name), op_type_name, + opts->op_registry()); + node_builder.Input(input); + *output = opts->FinalizeBuilder(&node_builder); + if (*output == nullptr) { + return errors::Internal("AddDataset: Failed to build ", op_type_name, + " op."); + } + return Status::OK(); + } + private: + void AddTensorInternal(const Tensor& val, Node** output) { + *output = ops::SourceOp( + "Const", + b_->opts().WithAttr("dtype", val.dtype()).WithAttr("value", val)); + } + bool HasAttr(const string& op_type_name, const string& attr_name) { const OpDef* op_def = nullptr; Status s = b_->opts().op_registry()->LookUpOpDef(op_type_name, &op_def); @@ -249,10 +267,6 @@ class IteratorContext { // range of outputs is typically represented by an `DatasetBase`, // defined below. class IteratorBase { - protected: - class IteratorBundleReader; - class IteratorBundleWriter; - public: virtual ~IteratorBase() {} @@ -284,87 +298,53 @@ class IteratorBase { virtual const std::vector& output_shapes() const = 0; // Saves the state of this iterator. - virtual Status Save(OpKernelContext* ctx, const string& path) { - BundleWriter bundle_writer(ctx->env(), path); - TF_RETURN_IF_ERROR(bundle_writer.status()); - IteratorBundleWriter writer(&bundle_writer); - TF_RETURN_IF_ERROR(Save(ctx, &writer)); - return bundle_writer.Finish(); + virtual Status Save(IteratorStateWriter* writer) { + if (is_exhausted_) { + LOG(INFO) << "Iterator exhausted."; + return writer->WriteScalar(kIteratorExhausted, kIteratorExhausted); + } else { + return SaveInternal(writer); + } } - virtual Status Restore(OpKernelContext* ctx, const string& path) { - if (!(ctx->env()->FileExists(MetaFilename(path)).ok())) { - return errors::NotFound( - "Failed to restore Iterator state. No file found at ", - MetaFilename(path)); + // Restores the state of this iterator. + virtual Status Restore(OpKernelContext* ctx, IteratorStateReader* reader) { + if (reader->Contains(kIteratorExhausted)) { + LOG(INFO) << "Iterator exhausted. Nothing to restore."; + is_exhausted_ = true; + return Status::OK(); + } else { + return RestoreInternal(ctx, reader); } - BundleReader bundle_reader(ctx->env(), path); - TF_RETURN_IF_ERROR(bundle_reader.status()); - IteratorBundleReader reader(&bundle_reader); - return Restore(ctx, &reader); } static const char kIteratorExhausted[]; protected: // This is needed so that sub-classes of IteratorBase can call - // `RestoreInternal` on their parent iterators, e.g., in + // `SaveInternal` on their parent iterators, e.g., in // `RepeatDataasetOp::Dataset`. - class IteratorBundleReader : public BundleReaderWrapper { - public: - IteratorBundleReader(BundleReader* bundle_reader) - : BundleReaderWrapper(bundle_reader) {} - - // Restores the state of a parent iterator recursively. - Status RestoreParent(OpKernelContext* ctx, - const std::unique_ptr& parent) { - return parent->RestoreInternal(ctx, this); - } - }; + Status SaveParent(IteratorStateWriter* writer, + const std::unique_ptr& parent) { + return parent->SaveInternal(writer); + } // This is needed so that sub-classes of IteratorBase can call - // `SaveInternal` on their parent iterators, e.g., in + // `RestoreInternal` on their parent iterators, e.g., in // `RepeatDataasetOp::Dataset`. - class IteratorBundleWriter : public BundleWriterWrapper { - public: - IteratorBundleWriter(BundleWriter* bundle_writer) - : BundleWriterWrapper(bundle_writer) {} - // Saves the state of a parent iterator recursively. - Status SaveParent(OpKernelContext* ctx, - const std::unique_ptr& parent) { - return parent->SaveInternal(ctx, this); - } - }; - - virtual Status Save(OpKernelContext* ctx, IteratorBundleWriter* writer) { - if (is_exhausted_) { - LOG(INFO) << "Iterator exhausted."; - return writer->WriteScalar(kIteratorExhausted, - kIteratorExhausted); - } else { - return SaveInternal(ctx, writer); - } + Status RestoreParent(OpKernelContext* ctx, IteratorStateReader* reader, + const std::unique_ptr& parent) { + return parent->RestoreInternal(ctx, reader); } - // Saves the state of this iterator. - virtual Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) { + // Saves the state of this iterator recursively. + virtual Status SaveInternal(IteratorStateWriter* writer) { return errors::Unimplemented("SaveInternal"); } - virtual Status Restore(OpKernelContext* ctx, IteratorBundleReader* reader) { - if (reader->Contains(kIteratorExhausted)) { - LOG(INFO) << "Iterator exhausted. Nothing to restore."; - is_exhausted_ = true; - return Status::OK(); - } else { - return RestoreInternal(ctx, reader); - } - } - - // Restores the state of this iterator. + // Restores the state of this iterator recursively. virtual Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) { + IteratorStateReader* reader) { return errors::Unimplemented("RestoreInternal"); } @@ -404,7 +384,7 @@ class DatasetBase : public core::RefCounted { virtual string DebugString() = 0; // Serializes the dataset and writes it to the `writer`. - virtual Status Save(BundleWriterWrapper* writer) const { + virtual Status Save(IteratorStateWriter* writer) const { return errors::Unimplemented("DatasetBase::Save"); } @@ -435,20 +415,14 @@ class GraphDatasetBase : public DatasetBase { const string op_name() const { return op_name_; } - Status Save(BundleWriterWrapper* writer) const override { - GraphDefBuilder b; - DatasetGraphDefBuilder db(&b); - Node* node = nullptr; - TF_RETURN_IF_ERROR(AsGraphDefInternal(&db, &node)); - string output_name = node->name(); - GraphDef graph_def; - TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); + Status Save(IteratorStateWriter* writer) const override { string serialized_graph_def; - graph_def.SerializeToString(&serialized_graph_def); + string output_node; + TF_RETURN_IF_ERROR(Serialize(&serialized_graph_def, &output_node)); TF_RETURN_IF_ERROR( - writer->WriteScalar(kDatasetGraphKey, serialized_graph_def)); + writer->WriteScalar(kDatasetGraphKey, serialized_graph_def)); TF_RETURN_IF_ERROR( - writer->WriteScalar(kDatasetGraphOutputNodeKey, output_name)); + writer->WriteScalar(kDatasetGraphOutputNodeKey, output_node)); return Status::OK(); } @@ -460,6 +434,18 @@ class GraphDatasetBase : public DatasetBase { static const char kDatasetGraphOutputNodeKey[]; private: + Status Serialize(string* serialized_graph_def, string* output_node) const { + GraphDefBuilder b; + DatasetGraphDefBuilder db(&b); + Node* node = nullptr; + TF_RETURN_IF_ERROR(AsGraphDefInternal(&db, &node)); + *output_node = node->name(); + GraphDef graph_def; + TF_RETURN_IF_ERROR(b.ToGraphDef(&graph_def)); + graph_def.SerializeToString(serialized_graph_def); + return Status::OK(); + } + const string op_name_; }; @@ -505,18 +491,18 @@ class DatasetIterator : public IteratorBase { return GetNextInternal(ctx, out_tensors, end_of_sequence); } - protected: - Status Save(OpKernelContext* ctx, IteratorBundleWriter* writer) final { + Status Save(IteratorStateWriter* writer) final { TF_RETURN_IF_ERROR(dataset()->Save(writer)); - return IteratorBase::Save(ctx, writer); + return IteratorBase::Save(writer); } + protected: // Internal implementation of GetNext that is wrapped in tracing logic. virtual Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, bool* end_of_sequence) = 0; - string full_name(const string& name) { + string full_name(const string& name) const { return strings::StrCat(prefix(), ":", name); } diff --git a/tensorflow/core/kernels/decode_csv_op.cc b/tensorflow/core/kernels/decode_csv_op.cc index 5e48ae976670de0c18293342cd5a313ed5c12018..c4555db453ba1549601cbf9a4bbf096fc3db22b2 100644 --- a/tensorflow/core/kernels/decode_csv_op.cc +++ b/tensorflow/core/kernels/decode_csv_op.cc @@ -91,9 +91,9 @@ class DecodeCSVOp : public OpKernel { } else { int32 value; OP_REQUIRES(ctx, strings::safe_strto32(fields[f], &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid int32: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid int32: ", + fields[f])); output[f]->flat()(i) = value; } break; @@ -111,9 +111,9 @@ class DecodeCSVOp : public OpKernel { } else { int64 value; OP_REQUIRES(ctx, strings::safe_strto64(fields[f], &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid int64: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid int64: ", + fields[f])); output[f]->flat()(i) = value; } break; @@ -130,13 +130,33 @@ class DecodeCSVOp : public OpKernel { } else { float value; OP_REQUIRES(ctx, strings::safe_strtof(fields[f].c_str(), &value), - errors::InvalidArgument( - "Field ", f, " in record ", i, - " is not a valid float: ", fields[f])); + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid float: ", + fields[f])); output[f]->flat()(i) = value; } break; } + case DT_DOUBLE: { + // If this field is empty or NA value, check if default is given: + // If yes, use default value; Otherwise report error. + if (fields[f].empty() || fields[f] == na_value_) { + OP_REQUIRES(ctx, record_defaults[f].NumElements() == 1, + errors::InvalidArgument( + "Field ", f, + " is required but missing in record ", i, "!")); + output[f]->flat()(i) = + record_defaults[f].flat()(0); + } else { + double value; + OP_REQUIRES(ctx, strings::safe_strtod(fields[f].c_str(), &value), + errors::InvalidArgument("Field ", f, " in record ", i, + " is not a valid double: ", + fields[f])); + output[f]->flat()(i) = value; + } + break; + } case DT_STRING: { // If this field is empty or NA value, check if default is given: // If yes, use default value; Otherwise report error. @@ -188,10 +208,9 @@ class DecodeCSVOp : public OpKernel { if (!quoted) { while (static_cast(current_idx) < input.size() && input[current_idx] != delim_) { - OP_REQUIRES(ctx, - (!use_quote_delim_ || input[current_idx] != '"') && - input[current_idx] != '\n' && - input[current_idx] != '\r', + OP_REQUIRES(ctx, (!use_quote_delim_ || input[current_idx] != '"') && + input[current_idx] != '\n' && + input[current_idx] != '\r', errors::InvalidArgument( "Unquoted fields cannot have quotes/CRLFs inside")); field += input[current_idx]; @@ -219,11 +238,10 @@ class DecodeCSVOp : public OpKernel { } OP_REQUIRES( - ctx, - (static_cast(current_idx) < input.size() && - input[current_idx] == '"' && - (static_cast(current_idx) == input.size() - 1 || - input[current_idx + 1] == delim_)), + ctx, (static_cast(current_idx) < input.size() && + input[current_idx] == '"' && + (static_cast(current_idx) == input.size() - 1 || + input[current_idx + 1] == delim_)), errors::InvalidArgument("Quoted field has to end with quote " "followed by delim or end")); diff --git a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc index 0174c8dfc8b3058c64fe4bd1279696eea282e4ae..e80d11eaea1640c54c21a7b94a2f043099c790f3 100644 --- a/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc +++ b/tensorflow/core/kernels/dense_to_sparse_batch_dataset_op.cc @@ -245,7 +245,6 @@ class DenseToSparseBatchDatasetOp : public UnaryDatasetOpKernel { private: mutex mu_; - int64 i_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc index 357c1f1be4e41ec46a2912a3669ec297a357492e..7a66285383368bb28dd3d0cd2fc6ff360eb82f5b 100644 --- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc +++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc @@ -67,7 +67,6 @@ __global__ void D2S_NCHW(const int32 nthreads, const int block_size, const int input_width, const int output_depth_by_input_height, dtype* __restrict__ output_ptr) { - // TODO(pauldonnelly): Implement more optimized kernels. CUDA_1D_KERNEL_LOOP(input_idx, nthreads) { // We will be converting the image from ordering: // n, bY, bX, oC, iY, iX (== input_idx) to @@ -99,6 +98,47 @@ __global__ void D2S_NCHW(const int32 nthreads, } } +template +__global__ void D2S_NCHW_LOOP(const int32 nthreads, + const dtype* __restrict__ input, + const int input_width, const int output_width, + const int output_depth_by_input_area, + const int input_depth_by_input_area, + dtype* __restrict__ output) { + CUDA_1D_KERNEL_LOOP(thread_idx, nthreads) { + // We will be converting the image from ordering: + // n, bY, bX, oC, iY, iX to + // n, oC, iY, bY, iX, bX + + // We assume thread_idx encodes n_oC_iY_iX, and use an unrolled loop over + // bY and bX coordinates within the block. This kernel is significantly + // more performant than the D2S_NCHW kernel. + // A likely explanation of the improvement is that although both kernels + // get input coalescing, this one would write the output data more densely + // per warp, so would benefit assuming delayed cache writeback is used. + + const int n_oC_iY = thread_idx / input_width; + const int iX = thread_idx - n_oC_iY * input_width; + + const int n = thread_idx / output_depth_by_input_area; + const int oC_iY_iX = thread_idx - n * output_depth_by_input_area; + + // Recombine the components and apply to the input and output pointers. + auto input_ptr = input + n * input_depth_by_input_area + oC_iY_iX; + auto output_ptr = output + (n_oC_iY * output_width + iX) * block_size; + +#pragma unroll + // Copy a patch of data to the output batch image. + for (int bY = 0; bY < block_size; ++bY) { +#pragma unroll + for (int bX = 0; bX < block_size; ++bX) { + output_ptr[bY * output_width + bX] = ldg( + input_ptr + (bY * block_size + bX) * output_depth_by_input_area); + } + } + } +} + } // namespace // Specialization of DepthToSpaceOpFunctor for a GPUDevice. @@ -139,10 +179,41 @@ struct DepthToSpaceOpFunctor { const int input_height = input.dimension(2); const int input_width = input.dimension(3); const int output_depth = output.dimension(1); - const int total_count = - batch_size * input_height * input_width * input_depth; + const int input_area = input_width * input_height; + const int input_depth_by_input_area = input_depth * input_area; + + // We improve performance by generating instantiations of the loop kernel + // for the most common block sizes. + if (block_size <= 4) { + const int output_width = output.dimension(3); + const int output_depth_by_input_area = output_depth * input_area; + const int total_count = batch_size * output_depth_by_input_area; + CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d); + switch (block_size) { + case 2: + return D2S_NCHW_LOOP + <<>>( + total_count, input.data(), input_width, output_width, + output_depth_by_input_area, input_depth_by_input_area, + output.data()); + case 3: + return D2S_NCHW_LOOP + <<>>( + total_count, input.data(), input_width, output_width, + output_depth_by_input_area, input_depth_by_input_area, + output.data()); + case 4: + return D2S_NCHW_LOOP + <<>>( + total_count, input.data(), input_width, output_width, + output_depth_by_input_area, input_depth_by_input_area, + output.data()); + } + } + + // Other block sizes are processed by the generic kernel. + const int total_count = batch_size * input_depth_by_input_area; auto config = GetCudaLaunchConfig(total_count, d); - D2S_NCHW<<>>( config.virtual_thread_count, input.data(), block_size, input_width, output_depth * input_height, output.data()); diff --git a/tensorflow/core/kernels/determinant_op.cc b/tensorflow/core/kernels/determinant_op.cc index 876dbff0301d30973cb72ddda37c3e6de10c84e1..b06f42384ebb54cce00009e06539f8ec9fc1dace 100644 --- a/tensorflow/core/kernels/determinant_op.cc +++ b/tensorflow/core/kernels/determinant_op.cc @@ -14,10 +14,13 @@ limitations under the License. ==============================================================================*/ // See docs in ../ops/linalg_ops.cc. + #include #if GOOGLE_CUDA #define EIGEN_USE_GPU +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/kernels/determinant_op.h" #endif #include "third_party/eigen3/Eigen/LU" @@ -31,23 +34,24 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #if GOOGLE_CUDA -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/fill_functor.h" #endif namespace tensorflow { -// A helper function to compute the sign and absolute value of the -// log of the determinant of inputs via a partially pivoted LU +// A helper function to compute the sign and absolute value of the log of the +// determinant of inputs via a partially pivoted LU // factorization. // -// Returns the sign in 'sign' and the log determinant in 'logdet' +// Returns the log of the absolute value of the determinant, and its sign in +// 'sign'. template -static void SLogDet( +static typename Eigen::NumTraits::Real SLogDet( const Eigen::Matrix& inputs, - Scalar* sign, Scalar* log_abs_det) { - *log_abs_det = 0; + Scalar* sign) { + using RealScalar = typename Eigen::NumTraits::Real; + RealScalar log_abs_det = 0; *sign = 1; // An empty matrix' determinant is defined to be 1. // (https://en.wikipedia.org/wiki/Determinant) @@ -58,27 +62,25 @@ static void SLogDet( Eigen::Matrix LU = lu.matrixLU(); *sign = lu.permutationP().determinant(); auto diag = LU.diagonal().array().eval(); - auto abs_diag = diag.cwiseAbs().template cast().eval(); - *log_abs_det += abs_diag.log().sum(); + auto abs_diag = diag.cwiseAbs().eval(); + log_abs_det += abs_diag.log().sum(); *sign *= (diag / abs_diag).prod(); } - if (!Eigen::numext::isfinite(*log_abs_det)) { + if (!Eigen::numext::isfinite(log_abs_det)) { *sign = 0; - *log_abs_det = std::log(0.0); + log_abs_det = + log_abs_det > 0 ? -std::log(RealScalar(0)) : std::log(RealScalar(0)); } + return log_abs_det; } template class LogDeterminantOp : public LinearAlgebraOp { public: - typedef LinearAlgebraOp Base; + INHERIT_LINALG_TYPEDEFS(Scalar); explicit LogDeterminantOp(OpKernelConstruction* context) : Base(context) {} - using TensorShapes = typename Base::TensorShapes; - using MatrixMaps = typename Base::MatrixMaps; - using ConstMatrixMaps = typename Base::ConstMatrixMaps; - TensorShapes GetOutputMatrixShapes( const TensorShapes& input_matrix_shapes) const final { return TensorShapes({TensorShape({}), TensorShape({})}); @@ -87,9 +89,9 @@ class LogDeterminantOp : public LinearAlgebraOp { void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs, MatrixMaps* outputs) final { Scalar sign; - Scalar log_abs_det; - SLogDet(Eigen::Matrix(inputs[0]), - &sign, &log_abs_det); + const RealScalar log_abs_det = SLogDet( + Eigen::Matrix(inputs[0]), + &sign); outputs->at(0)(0, 0) = sign; outputs->at(1)(0, 0) = log_abs_det; @@ -99,14 +101,10 @@ class LogDeterminantOp : public LinearAlgebraOp { template class DeterminantOp : public LinearAlgebraOp { public: - typedef LinearAlgebraOp Base; + INHERIT_LINALG_TYPEDEFS(Scalar); explicit DeterminantOp(OpKernelConstruction* context) : Base(context) {} - using TensorShapes = typename Base::TensorShapes; - using MatrixMaps = typename Base::MatrixMaps; - using ConstMatrixMaps = typename Base::ConstMatrixMaps; - TensorShapes GetOutputMatrixShapes( const TensorShapes& input_matrix_shape) const final { return TensorShapes({TensorShape({})}); @@ -115,15 +113,10 @@ class DeterminantOp : public LinearAlgebraOp { void ComputeMatrix(OpKernelContext* context, const ConstMatrixMaps& inputs, MatrixMaps* outputs) final { Scalar sign; - Scalar log_abs_det; - SLogDet(Eigen::Matrix(inputs[0]), - &sign, &log_abs_det); - Scalar determinant = sign * std::exp(log_abs_det); - // TODO(rmlarsen): Don't fail on infinite determinants, since that could - // be a valid result and the user should check for it instead. - OP_REQUIRES(context, Eigen::numext::isfinite(determinant), - errors::InvalidArgument("The determinant is not finite.")); - outputs->at(0)(0, 0) = determinant; + const RealScalar log_abs_det = SLogDet( + Eigen::Matrix(inputs[0]), + &sign); + outputs->at(0)(0, 0) = sign * std::exp(log_abs_det); } }; @@ -171,7 +164,7 @@ class DeterminantOpGpu : public AsyncOpKernel { return; } - // TODO(rmlarsen): Convert to std::make_unique when available. + // TODO(rmlarsen): Convert to absl::make_unique when available. std::unique_ptr solver(new CudaSolver(context)); // Reuse the input buffer or make a copy for the factorization step, @@ -255,18 +248,160 @@ class DeterminantOpGpu : public AsyncOpKernel { for (int i = 0; i < host_infos[0].size(); ++i) { // It is OK for a matrix to be singular (signaled by info > 0), // corresponding to determinant of zero, but we do want to catch - // invalid arguments to GetrfBatched. + // invalid arguments to Getrf{Batched}. OP_REQUIRES_ASYNC( - context, - host_infos[0].data()[i] >= 0 || - host_infos[0].data()[i] == kint32min, + context, host_infos[0](i) >= 0, errors::InvalidArgument("Invalid input argument no. ", host_infos[0].data()[i], " for batch index ", i, "."), done); + } + } + done(); + }; + CudaSolver::CheckLapackInfoAndDeleteSolverAsync(std::move(solver), dev_info, + std::move(info_checker)); + } +}; + +template +class LogDeterminantOpGpu : public AsyncOpKernel { + public: + explicit LogDeterminantOpGpu(OpKernelConstruction* context) + : AsyncOpKernel(context) {} + + void ComputeAsync(OpKernelContext* context, DoneCallback done) final { + const Tensor& input = context->input(0); + const int ndims = input.dims(); + const int64 n = input.dim_size(ndims - 1); + // Validate inputs. + OP_REQUIRES_ASYNC( + context, ndims >= 2, + errors::InvalidArgument("Input must have rank >= 2, got ", ndims), + done); + OP_REQUIRES_ASYNC( + context, input.dim_size(ndims - 2) == n, + errors::InvalidArgument("Input matrices must be square, got", + input.dim_size(ndims - 2), " != ", n), + done); + + // Allocate output. + TensorShape out_shape; + for (int dim = 0; dim < ndims - 2; ++dim) { + out_shape.AddDim(input.dim_size(dim)); + } + out_shape.AppendShape(TensorShape({})); + Tensor* sign; + OP_REQUIRES_OK_ASYNC(context, context->allocate_output(0, out_shape, &sign), + done); + Tensor* log_abs_det; + OP_REQUIRES_OK_ASYNC( + context, context->allocate_output(1, out_shape, &log_abs_det), done); + + // By definition, the determinant of an empty matrix is equal to one. + const GPUDevice& d = context->eigen_device(); + if (input.NumElements() == 0) { + functor::SetOneFunctor one_func; + one_func(d, sign->template flat()); + functor::SetZeroFunctor zero_func; + zero_func(d, log_abs_det->template flat()); + done(); + return; + } + + // TODO(rmlarsen): Convert to absl::make_unique when available. + std::unique_ptr solver(new CudaSolver(context)); + + // Reuse the input buffer or make a copy for the factorization step, + // depending on whether this ops owns it exclusively. + Tensor input_copy; + OP_REQUIRES_OK_ASYNC( + context, + solver->forward_input_or_allocate_scoped_tensor( + {0}, DataTypeToEnum::value, input.shape(), &input_copy), + done); + if (!input.SharesBufferWith(input_copy)) { + d.memcpy(input_copy.flat().data(), input.flat().data(), + input.NumElements() * sizeof(Scalar)); + } + auto input_copy_reshaped = input_copy.template flat_inner_dims(); + const int64 batch_size = input_copy_reshaped.dimension(0); + + // Allocate pivots on the device. + Tensor pivots; + OP_REQUIRES_OK_ASYNC( + context, + solver->allocate_scoped_tensor(DataTypeToEnum::value, + TensorShape{batch_size, n}, &pivots), + done); + auto pivots_mat = pivots.template matrix(); + + // Prepare pointer arrays for cuBlas' batch interface. + // TODO(rmlarsen): Find a way to encode pointer arrays in pinned host memory + // without the ugly casting. + auto input_copy_ptrs = solver->GetScratchSpace( + sizeof(Scalar*) * batch_size, "input_copy_ptrs", + /* on_host */ true); + + // Compute the partially pivoted LU factorization(s) of the matrix/matrices. + std::vector dev_info; + if (n / batch_size <= 128) { + // For small matrices or large batch sizes, we use the batched interface + // from cuBlas. + const Scalar** input_copy_ptrs_base = + reinterpret_cast(input_copy_ptrs.mutable_data()); + for (int batch = 0; batch < batch_size; ++batch) { + input_copy_ptrs_base[batch] = &input_copy_reshaped(batch, 0, 0); + } + dev_info.push_back( + solver->GetDeviceLapackInfo(batch_size, "getrfBatched")); + OP_REQUIRES_OK_ASYNC( + context, + solver->GetrfBatched(n, input_copy_ptrs_base, n, pivots_mat.data(), + &dev_info.back(), batch_size), + done); + } else { + // For large matrices or small batch sizes we use the non-batched + // interface from cuSolver, which is much faster for large matrices. + dev_info.push_back(solver->GetDeviceLapackInfo(batch_size, "getrf")); + for (int batch = 0; batch < batch_size; ++batch) { + OP_REQUIRES_OK_ASYNC( + context, + solver->Getrf(n, n, &input_copy_reshaped(batch, 0, 0), n, + &pivots_mat(batch, 0), &dev_info.back()(batch)), + done); + } + } + + auto input_copy_reshaped_const = + const_cast(&input_copy) + ->template flat_inner_dims(); + auto sign_reshaped = sign->flat(); + auto log_abs_det_reshaped = log_abs_det->flat(); + // Compute the determinant for each batch as (-1)^s * prod(diag(U)), + // where s is the order of the permutation encoded in pivots and U is the + // upper triangular factor of the LU factorization, which is written to + // input_copy by the Getrf{Batched} kernel. + functor::LogDeterminantFromPivotedLUFunctor functor; + functor(d, input_copy_reshaped_const, pivots_mat.data(), sign_reshaped, + log_abs_det_reshaped); + + // Register callback to check info after kernels finish. + auto info_checker = [context, done]( + const Status& status, + const std::vector& host_infos) { + if (!status.ok() && errors::IsInvalidArgument(status) && + !host_infos.empty()) { + for (int i = 0; i < host_infos[0].size(); ++i) { + // It is OK for a matrix to be singular (signaled by info > 0), + // corresponding to determinant of zero, but we do want to catch + // invalid arguments to Getrf{Batched}. OP_REQUIRES_ASYNC( - context, host_infos[0].data()[i] != kint32min, - errors::InvalidArgument("The determinant is not finite."), done); + context, host_infos[0](i) >= 0, + errors::InvalidArgument("Invalid input argument no. ", + host_infos[0].data()[i], + " for batch index ", i, "."), + done); } } done(); @@ -282,6 +417,15 @@ REGISTER_LINALG_OP_GPU("MatrixDeterminant", (DeterminantOpGpu), complex64); REGISTER_LINALG_OP_GPU("MatrixDeterminant", (DeterminantOpGpu), complex128); + +REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu), + float); +REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu), + double); +REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", (LogDeterminantOpGpu), + complex64); +REGISTER_LINALG_OP_GPU("LogMatrixDeterminant", + (LogDeterminantOpGpu), complex128); #endif // GOOGLE_CUDA REGISTER_LINALG_OP("MatrixDeterminant", (DeterminantOp), float); diff --git a/tensorflow/core/kernels/determinant_op.h b/tensorflow/core/kernels/determinant_op.h new file mode 100644 index 0000000000000000000000000000000000000000..e931e328e4bbb2e29f3f3ff4fbaf3dfb76fb1ea7 --- /dev/null +++ b/tensorflow/core/kernels/determinant_op.h @@ -0,0 +1,47 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_ +#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_ + +#include "tensorflow/core/framework/tensor_types.h" + +namespace tensorflow { +namespace functor { + +// Helper functor to compute Determinant from a partially pivoted LU +// factorization. +template +struct DeterminantFromPivotedLUFunctor { + void operator()(const Device& device, + typename TTypes::ConstTensor lu_factor, + const int* pivots, typename TTypes::Tensor output, + int* info); +}; + +// Helper functor to compute sign and log of the absolute value of the +// determinant from a partially pivoted LU factorization. +template +struct LogDeterminantFromPivotedLUFunctor { + void operator()(const Device& device, + typename TTypes::ConstTensor lu_factor, + const int* pivots, typename TTypes::Tensor sign, + typename TTypes::Tensor log_abs_det); +}; + +} // namespace functor +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_ diff --git a/tensorflow/core/kernels/determinant_op_gpu.cu.cc b/tensorflow/core/kernels/determinant_op_gpu.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..c866204c97e6acd160f1b4ec1eed989d88c52eff --- /dev/null +++ b/tensorflow/core/kernels/determinant_op_gpu.cu.cc @@ -0,0 +1,168 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include "tensorflow/core/kernels/determinant_op.h" + +#include +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" + +namespace tensorflow { +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; +namespace { +__device__ int PermutationOrder(int n, const int* pivots) { + // Compute the order of the permutation from the number of transpositions + // encoded in the pivot array, see: + // http://icl.cs.utk.edu/lapack-forum/viewtopic.php?f=2&t=340 + int order = 0; + for (int i = 0; i < n - 1; ++i) { + // Notice: Internally, the cuBlas code uses Fortran convention (1-based) + // indexing so we expect pivots[i] == i + 1 for rows that were not moved. + order += pivots[i] != (i + 1); + } + return order; +} + +#if defined(__CUDACC__) +// Hack around missing support for complex in NVCC. +template +__device__ inline std::complex complex_multiply(const std::complex& a, + const std::complex& b) { + const T a_real = Eigen::numext::real(a); + const T a_imag = Eigen::numext::imag(a); + const T b_real = Eigen::numext::real(b); + const T b_imag = Eigen::numext::imag(b); + return std::complex(a_real * b_real - a_imag * b_imag, + a_real * b_imag + a_imag * b_real); +} +__device__ inline complex64 operator*(const complex64& a, const complex64& b) { + return complex_multiply(a, b); +} +__device__ inline complex64 operator*(const complex64& a, const float& b) { + return complex64(Eigen::numext::real(a) * b, Eigen::numext::imag(a) * b); +} +__device__ inline complex64 operator/(const complex64& a, const float& b) { + const float inv_b = 1.0f / b; + return a * inv_b; +} +__device__ inline complex128 operator*(const complex128& a, + const complex128& b) { + return complex_multiply(a, b); +} +__device__ inline complex128 operator*(const complex128& a, const double& b) { + return complex128(Eigen::numext::real(a) * b, Eigen::numext::imag(a) * b); +} +__device__ inline complex128 operator/(const complex128& a, const double& b) { + const double inv_b = 1.0 / b; + return a * inv_b; +} +#endif +} // namespace + +// This kernel computes either determinant or log_abs_determinant, depending +// on the value of the template parameter. If compute_log_abs_det is false, +// the sign argument is ignored. +template +__global__ void DeterminantFromPivotedLUKernel(int nthreads, int n, + const Scalar* lu_factor, + const int* all_pivots, + Scalar* sign, + Scalar* log_abs_det) { + typedef typename Eigen::NumTraits::Real RealScalar; + const int matrix_size = n * n; + const int stride = n + 1; + // We only parallelize over batches here. Performance is not critical, + // since this cheap O(n) kernel always follows an O(n^3) LU factorization. + // The main purpose is to avoid having to copy the LU decomposition to + // host memory. + CUDA_1D_KERNEL_LOOP(o_idx, nthreads) { + // Initialize sign to (-1)^order. + const int order = PermutationOrder(n, all_pivots + o_idx * n); + Scalar prod_sign = order % 2 ? Scalar(-1) : Scalar(1); + RealScalar sum_log_abs_det = RealScalar(0); + int i_idx = matrix_size * o_idx; + for (int i = 0; i < n; ++i, i_idx += stride) { + const RealScalar abs_i = Eigen::numext::abs(lu_factor[i_idx]); + sum_log_abs_det += Eigen::numext::log(abs_i); + prod_sign = prod_sign * (lu_factor[i_idx] / abs_i); + } + if (!Eigen::numext::isfinite(sum_log_abs_det)) { + prod_sign = Scalar(0); + sum_log_abs_det = sum_log_abs_det > 0 ? -Eigen::numext::log(RealScalar(0)) + : Eigen::numext::log(RealScalar(0)); + } + if (compute_log_abs_det) { + sign[o_idx] = prod_sign; + log_abs_det[o_idx] = Scalar(sum_log_abs_det); + } else { + log_abs_det[o_idx] = prod_sign * Eigen::numext::exp(sum_log_abs_det); + } + } +} + +template +struct DeterminantFromPivotedLUFunctor { + void operator()(const GPUDevice& device, + typename TTypes::ConstTensor lu_factor, + const int* pivots, typename TTypes::Tensor output, + int* info) { + const int64 num_matrices = output.size(); + const int64 n = lu_factor.dimension(2); + CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device); + DeterminantFromPivotedLUKernel + <<>>( + config.virtual_thread_count, n, lu_factor.data(), pivots, nullptr, + output.data()); + } +}; + +template struct DeterminantFromPivotedLUFunctor; +template struct DeterminantFromPivotedLUFunctor; +template struct DeterminantFromPivotedLUFunctor; +template struct DeterminantFromPivotedLUFunctor; + +template +struct LogDeterminantFromPivotedLUFunctor { + void operator()(const GPUDevice& device, + typename TTypes::ConstTensor lu_factor, + const int* pivots, typename TTypes::Tensor sign, + typename TTypes::Tensor log_abs_det) { + const int64 num_matrices = sign.size(); + const int64 n = lu_factor.dimension(2); + CudaLaunchConfig config = GetCudaLaunchConfig(num_matrices, device); + DeterminantFromPivotedLUKernel + <<>>( + config.virtual_thread_count, n, lu_factor.data(), pivots, + sign.data(), log_abs_det.data()); + } +}; + +template struct LogDeterminantFromPivotedLUFunctor; +template struct LogDeterminantFromPivotedLUFunctor; +template struct LogDeterminantFromPivotedLUFunctor; +template struct LogDeterminantFromPivotedLUFunctor; + +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/diag_op.cc b/tensorflow/core/kernels/diag_op.cc index c800859d90366d8bd2139120401a2a1a42bb71c7..be862b82f1b311e3e46bbe27de9921bb548fa0b6 100644 --- a/tensorflow/core/kernels/diag_op.cc +++ b/tensorflow/core/kernels/diag_op.cc @@ -14,65 +14,32 @@ limitations under the License. ==============================================================================*/ // See docs in ../ops/array_ops.cc + +#define EIGEN_USE_THREADS + +#if GOOGLE_CUDA +#define EIGEN_USE_GPU +#endif // GOOGLE_CUDA + +#include "tensorflow/core/kernels/diag_op.h" + +#include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/work_sharder.h" namespace tensorflow { -namespace { -template -class DiagonalGenerator { - public: - explicit DiagonalGenerator(const Tensor& diagonal) : diagonal_(diagonal) { - static_assert(DoubleNumDims == 2 * NumDims, - "The second size must be the double of the first size."); - CHECK_EQ(diagonal.dims(), NumDims); - } - T operator()( - const Eigen::array& coordinates) const { - Eigen::array index; - for (size_t i = 0; i < NumDims; ++i) { - if (coordinates[i] != coordinates[NumDims + i]) { - return T(0); - } - index[i] = coordinates[i]; - } - return diagonal_.tensor()(index); - } - private: - Tensor diagonal_; -}; - -template -class DiagonalExtractor { - public: - explicit DiagonalExtractor(const Tensor& tensor) : tensor_(tensor) { - CHECK_EQ(tensor.dims(), 2 * NumDims); - } - T operator()(const Eigen::array& coordinates) const { - Eigen::array index; - for (size_t j = 0; j < NumDims; ++j){ - index[j] = coordinates[j]; - } - for (size_t j = NumDims; j < 2 * NumDims; ++j){ - index[j] = index[j - NumDims]; - } - return tensor_.tensor()(index); - } - - private: - Tensor tensor_; -}; - -} // namespace +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef Eigen::GpuDevice GPUDevice; // Generate the diagonal tensor with the diagonal set to the input tensor. -// It only allows up to rank 3 input tensor, so the output tensor is up to -// rank 6. -template +template class DiagOp : public OpKernel { public: explicit DiagOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -80,9 +47,8 @@ class DiagOp : public OpKernel { void Compute(OpKernelContext* context) override { const Tensor& diagonal = context->input(0); const int num_dims = diagonal.dims(); - OP_REQUIRES(context, 1 <= num_dims && num_dims <= 3, - errors::InvalidArgument("Expected 1 <= dims <= 3, got shape ", - diagonal.shape().DebugString())); + OP_REQUIRES(context, 0 != num_dims, errors::InvalidArgument( + "Input must be at least rank 1, got 0")); TensorShape out_shape; for (int i = 0; i < num_dims; ++i) { out_shape.AddDim(diagonal.dim_size(i)); @@ -93,45 +59,17 @@ class DiagOp : public OpKernel { Tensor* output_tensor = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output_tensor)); - switch (num_dims) { - case 1: - output_tensor->tensor() = output_tensor->tensor().generate( - DiagonalGenerator(diagonal)); - break; - case 2: - output_tensor->tensor() = output_tensor->tensor().generate( - DiagonalGenerator(diagonal)); - break; - case 3: - output_tensor->tensor() = output_tensor->tensor().generate( - DiagonalGenerator(diagonal)); - break; - default: - context->SetStatus(errors::Unimplemented( - "Diagonal of rank ", num_dims, " tensor is not supported yet.")); - return; - } + functor::DiagFunctor diagFunc; + Status s = diagFunc(context, + diagonal.NumElements(), + diagonal.flat().data(), + output_tensor->flat().data()); + OP_REQUIRES_OK(context, s); } }; -#define REGISTER_DIAGOP(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("Diag").Device(DEVICE_CPU).TypeConstraint("T"), DiagOp) - -REGISTER_DIAGOP(double); -REGISTER_DIAGOP(float); -REGISTER_DIAGOP(int32); -REGISTER_DIAGOP(int64); -REGISTER_DIAGOP(complex64); -REGISTER_DIAGOP(complex128); - -#undef REGISTER_DIAGOP - - -// Generate the diagonal tensor with the diagonal set to the input tensor. -// It only allows rank 2, 4, or 6 input tensor, so the output tensor is -// rank 1, 2, or 3. -template +// Extract the diagonal tensor with the diagonal set to the input tensor. +template class DiagPartOp : public OpKernel { public: explicit DiagPartOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -140,9 +78,9 @@ class DiagPartOp : public OpKernel { const Tensor& tensor = context->input(0); const int num_dims = tensor.dims(); const int out_dims = num_dims / 2; - OP_REQUIRES(context, 2 == num_dims || 4 == num_dims || 6 == num_dims, - errors::InvalidArgument("The rank of the tensor should be 2, \ - 4, or 6, got shape ", + OP_REQUIRES(context, 0 == num_dims % 2, + errors::InvalidArgument("The rank of the tensor should be \ + even and positive, got shape ", tensor.shape().DebugString())); for (int i = 0; i < out_dims; i++){ OP_REQUIRES(context, tensor.dim_size(i) == tensor.dim_size(i + out_dims), @@ -160,39 +98,158 @@ class DiagPartOp : public OpKernel { Tensor* output = nullptr; OP_REQUIRES_OK(context, context->allocate_output(0, out_shape, &output)); + functor::DiagPartFunctor diagPartFunc; + Status s = diagPartFunc(context, + out_shape.num_elements(), + tensor.flat().data(), + output->flat().data()); + OP_REQUIRES_OK(context, s); + } +}; - switch (num_dims) { - case 2: - output->tensor() = output->tensor().generate( - DiagonalExtractor(tensor)); - break; - case 4: - output->tensor() = output->tensor().generate( - DiagonalExtractor(tensor)); - break; - case 6: - output->tensor() = output->tensor().generate( - DiagonalExtractor(tensor)); - break; - default: - context->SetStatus(errors::Unimplemented( - "Diagonal of rank ", num_dims, " tensor is not supported yet.")); - return; - } +// Implementation of the functor specialization for CPU. +// +// According to the diagonal definition, +// `output[i1,..., ik, i1,..., ik] = input[i1,..., ik]`, +// +// Let the rank of input is [s1,..., sk], then any offset of input's +// pointer can be represent by coordinate [i1,..., ik], +// where `index = i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik` +// +// Let new_index is the offset of output's pointer with coordinate +// [i1,..., ik, i1,..., ik], then we have +// `new_index = i1*(s2*...sk*s1*...*sk) + i2*(s3*...*sk*s1*...*sk) +... + \ +// ik*(s1*...*sk) + i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik +// = (i1*(s2*...*sk) + i2*(s3*...*sk) +... + ik) * (1 + s1*...*sk) +// = index * (1 + s1*...*sk) +// +// Let `size = s1*...*sk`, we finally have `new_index = index * (1 + size)`, +// which is the transfer function we use below. +// This trick make our implementations clear and easy to be parallel. +namespace functor { +template +struct DiagFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // This subprocess is responsible for writing values in index range + // [start*size, limit*size) + auto subDiag = [in, out, size](int64 start, int64 limit) { + std::fill(out + size * start, out + size * limit, T()); + for (int64 index = start; index < limit; ++index) { + out[(1 + size) * index] = in[index]; + } + }; + + // Here, 5 is a empirical factor of cost_per_unit. + auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); + Shard(worker_threads.num_threads, worker_threads.workers, size, + 5 * size, subDiag); + return Status::OK(); + } +}; + +template +struct DiagPartFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // This subprocess is responsible for extracting values in index range + // [start, limit) + auto subDiagPart = [in, out, size](int64 start, int64 limit) { + for (int64 index = start; index < limit; ++index) { + out[index] = in[(1 + size) * index]; + } + }; + + // Here, 5 is a empirical factor of cost_per_unit. + auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); + Shard(worker_threads.num_threads, worker_threads.workers, size, + 5, subDiagPart); + return Status::OK(); } }; +} // namespace functor -#define REGISTER_DIAGPARTOP(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("DiagPart").Device(DEVICE_CPU).TypeConstraint("T"), DiagPartOp) -REGISTER_DIAGPARTOP(double); -REGISTER_DIAGPARTOP(float); -REGISTER_DIAGPARTOP(int32); -REGISTER_DIAGPARTOP(int64); -REGISTER_DIAGPARTOP(complex64); -REGISTER_DIAGPARTOP(complex128); +// Register the CPU kernels. +#define REGISTER_DIAGOP(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("Diag").Device(DEVICE_CPU).TypeConstraint("T"), \ + DiagOp) +TF_CALL_double(REGISTER_DIAGOP); +TF_CALL_float(REGISTER_DIAGOP); +TF_CALL_int32(REGISTER_DIAGOP); +TF_CALL_int64(REGISTER_DIAGOP); +TF_CALL_complex64(REGISTER_DIAGOP); +TF_CALL_complex128(REGISTER_DIAGOP); +#undef REGISTER_DIAGOP + +#define REGISTER_DIAGPARTOP(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("DiagPart").Device(DEVICE_CPU).TypeConstraint("T"), \ + DiagPartOp) + +TF_CALL_double(REGISTER_DIAGPARTOP); +TF_CALL_float(REGISTER_DIAGPARTOP); +TF_CALL_int32(REGISTER_DIAGPARTOP); +TF_CALL_int64(REGISTER_DIAGPARTOP); +TF_CALL_complex64(REGISTER_DIAGPARTOP); +TF_CALL_complex128(REGISTER_DIAGPARTOP); #undef REGISTER_DIAGPARTOP - + +// Register the GPU kernels. +#ifdef GOOGLE_CUDA + +// Forward declarations of the functor specializations for GPU. +namespace functor { +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +extern template struct DiagFunctor; +} // namespace functor + +#define REGISTER_DIAGOP_GPU(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("Diag").Device(DEVICE_GPU).TypeConstraint("T"), \ + DiagOp) + +TF_CALL_double(REGISTER_DIAGOP_GPU); +TF_CALL_float(REGISTER_DIAGOP_GPU); +TF_CALL_int32(REGISTER_DIAGOP_GPU); +TF_CALL_int64(REGISTER_DIAGOP_GPU); +TF_CALL_complex64(REGISTER_DIAGOP_GPU); +TF_CALL_complex128(REGISTER_DIAGOP_GPU); +#undef REGISTER_DIAGOP_GPU + +// Forward declarations of the functor specializations for GPU. +namespace functor { +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +extern template struct DiagPartFunctor; +} // namespace functor + +#define REGISTER_DIAGPARTOP_GPU(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("DiagPart").Device(DEVICE_GPU).TypeConstraint("T"), \ + DiagPartOp) + +TF_CALL_double(REGISTER_DIAGPARTOP_GPU); +TF_CALL_float(REGISTER_DIAGPARTOP_GPU); +TF_CALL_int32(REGISTER_DIAGPARTOP_GPU); +TF_CALL_int64(REGISTER_DIAGPARTOP_GPU); +TF_CALL_complex64(REGISTER_DIAGPARTOP_GPU); +TF_CALL_complex128(REGISTER_DIAGPARTOP_GPU); +#undef REGISTER_DIAGPARTOP_GPU + +#endif // GOOGLE_CUDA + + } // namespace tensorflow + diff --git a/tensorflow/core/kernels/diag_op.h b/tensorflow/core/kernels/diag_op.h new file mode 100644 index 0000000000000000000000000000000000000000..c6ca6a2047455649b5197da27a58cb068476e928 --- /dev/null +++ b/tensorflow/core/kernels/diag_op.h @@ -0,0 +1,43 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_DIAG_OP_H_ +#define TENSORFLOW_CORE_KERNELS_DIAG_OP_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +namespace functor { + +template +struct DiagFunctor { + Status operator() (OpKernelContext* context, const int64 size, + const T* in, T* out); +}; + +template +struct DiagPartFunctor { + Status operator() (OpKernelContext* context, const int64 size, + const T* in, T* out); +}; + +} // namespace functor + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_DIAG_OP_H_ diff --git a/tensorflow/core/kernels/diag_op_gpu.cu.cc b/tensorflow/core/kernels/diag_op_gpu.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..684f00ea61d136a3ed75d6a6b19f7eff02c30d1e --- /dev/null +++ b/tensorflow/core/kernels/diag_op_gpu.cu.cc @@ -0,0 +1,139 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" +#include "tensorflow/core/kernels/diag_op.h" + +namespace tensorflow { +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; + +template +__global__ void DiagCudaKernel(const int num_threads, + const int64 size, + const T* in, + T* out) { + CUDA_1D_KERNEL_LOOP(index, num_threads) { + // Fill the diagonal elements or set to zero in other place. + if (index % (1 + size) == 0) { + out[index] = in[index / (1 + size)]; + } else { + out[index] = T(0); + } + } +} + +template +struct DiagFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // Empty tensor couldn't launch the kernel. + if (size == 0) { + return Status::OK(); + } + + // CudaLaunchConfig uses an int for virtual_thread_count, + // so this may overflow for `size*size` in extreme cases, + // here is checking the multiplication overflow for integer. + if (size && (int(size * size) / size) != size) { + return errors::Internal( + "DiagOp got input size too large."); + } + int virtual_thread_count = int(size * size); + + // Launch the GPU kernel. + const GPUDevice& device = context->eigen_device(); + CudaLaunchConfig diag_config = GetCudaLaunchConfig( + virtual_thread_count, device); + DiagCudaKernel<<>>( + diag_config.virtual_thread_count, size, in, out); + + auto err = cudaGetLastError(); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch DiagOp kernel: ", + cudaGetErrorString(err), "."); + } + return Status::OK(); + } +}; + +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; +template struct DiagFunctor; + + +template +__global__ void DiagPartCudaKernel(const int num_threads, + const int64 size, + const T* in, + T* out) { + CUDA_1D_KERNEL_LOOP(index, num_threads) { + out[index] = in[(1 + size) * index]; + } +} + +template +struct DiagPartFunctor { + EIGEN_ALWAYS_INLINE Status + operator() (OpKernelContext* context, const int64 size, + const T* in, T* out) { + // Empty tensor couldn't launch the kernel. + if (size == 0) { + return Status::OK(); + } + const GPUDevice& device = context->eigen_device(); + + // Extract the diagonal elements. + CudaLaunchConfig diag_config = GetCudaLaunchConfig(size, device); + DiagPartCudaKernel<<>>( + diag_config.virtual_thread_count, size, in, out); + + auto err = cudaGetLastError(); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch DiagPartOp kernel: ", + cudaGetErrorString(err), "."); + } + return Status::OK(); + } +}; + +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; +template struct DiagPartFunctor; + +} // end namespace functor +} // end namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/diag_op_test.cc b/tensorflow/core/kernels/diag_op_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..2d1417854cc06a138a803169495196ac70e70e5d --- /dev/null +++ b/tensorflow/core/kernels/diag_op_test.cc @@ -0,0 +1,54 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/test_benchmark.h" + +namespace tensorflow { + +template +static Graph* Diag(int n, DataType type) { + Graph* g = new Graph(OpRegistry::Global()); + Tensor in(type, TensorShape({n})); + in.flat().setRandom(); + Node* out = test::graph::Diag(g, test::graph::Constant(g, in), type); + test::graph::DiagPart(g, out, type); + return g; +} + +#define BM_DiagDev(N, T, TFTYPE, DEVICE) \ + static void BM_Diag##_##N##_##TFTYPE##_##DEVICE(int iters) { \ + testing::UseRealTime(); \ + testing::ItemsProcessed(static_cast(iters) * N * N); \ + test::Benchmark(#DEVICE, Diag(N, TFTYPE)).Run(iters); \ + } \ + BENCHMARK(BM_Diag##_##N##_##TFTYPE##_##DEVICE); + +#define BM_Diag(N) \ + BM_DiagDev(N, int, DT_INT32, cpu); \ + BM_DiagDev(N, float, DT_FLOAT, cpu); \ + BM_DiagDev(N, std::complex, DT_COMPLEX64, cpu); \ + BM_DiagDev(N, int, DT_INT32, gpu); \ + BM_DiagDev(N, float, DT_FLOAT, gpu); \ + BM_DiagDev(N, std::complex, DT_COMPLEX64, gpu); + +BM_Diag(16); +BM_Diag(128); +BM_Diag(512); + +} // end namespace tensorflow + diff --git a/tensorflow/core/kernels/extract_image_patches_op.h b/tensorflow/core/kernels/extract_image_patches_op.h index 2abbed15e5d3bdf1ae721f0071a3d46b2a88c726..e430a23d206c69c82495b78d87e64c70c1b0eaeb 100644 --- a/tensorflow/core/kernels/extract_image_patches_op.h +++ b/tensorflow/core/kernels/extract_image_patches_op.h @@ -32,11 +32,21 @@ struct ExtractImagePatchesForward { typename TTypes::Tensor output) { // Need to swap row/col when calling Eigen, because our data is in // NHWC format while Eigen assumes NWHC format. - To32Bit(output).device(d) = - To32Bit(input) - .extract_image_patches(patch_cols, patch_rows, stride_cols, - stride_rows, rate_cols, rate_rows, padding) - .reshape(output.dimensions()); + const int64 N = std::max(input.size(), output.size()); + if (N <= std::numeric_limits::max()) { + auto output_32bit = To32Bit(output); + output_32bit.device(d) = + To32Bit(input) + .extract_image_patches(patch_cols, patch_rows, stride_cols, + stride_rows, rate_cols, rate_rows, padding) + .reshape(output_32bit.dimensions()); + } else { + output.device(d) = + input + .extract_image_patches(patch_cols, patch_rows, stride_cols, + stride_rows, rate_cols, rate_rows, padding) + .reshape(output.dimensions()); + } } }; diff --git a/tensorflow/core/kernels/eye_functor.h b/tensorflow/core/kernels/eye_functor.h new file mode 100644 index 0000000000000000000000000000000000000000..70f093f81366e017f3a07614e319435e1bf5aca2 --- /dev/null +++ b/tensorflow/core/kernels/eye_functor.h @@ -0,0 +1,32 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ +#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ + +#include "tensorflow/core/framework/tensor_types.h" + +namespace tensorflow { +namespace functor { + +template +struct EyeFunctor { + void operator()(const Device& device, + typename TTypes::Tensor matrix_batch); +}; + +} // namespace functor +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ diff --git a/tensorflow/core/kernels/eye_functor_gpu.cu.cc b/tensorflow/core/kernels/eye_functor_gpu.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..a620316e27595aaa246b018a9a8afc6f678f45a4 --- /dev/null +++ b/tensorflow/core/kernels/eye_functor_gpu.cu.cc @@ -0,0 +1,70 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include "tensorflow/core/kernels/eye_functor.h" + +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" + +namespace tensorflow { +namespace functor { + +typedef Eigen::GpuDevice GPUDevice; + +template +__global__ void EyeKernel(int num_threads, int batch_size, int m, int n, + Scalar* output_ptr) { + const Scalar one = Scalar(1); + const Scalar zero = Scalar(0); + CUDA_1D_KERNEL_LOOP(index, num_threads) { + // TODO(rmlarsen): Benchmark to see if it's just as fast to use mod (%), + // since it's easier to read. + const int global_row = index / n; + const int col = index - global_row * n; + const int batch = global_row / m; + const int row = global_row - batch * m; + output_ptr[index] = col == row ? one : zero; + } +} + +template +struct EyeFunctor { + void operator()(const GPUDevice& device, + typename TTypes::Tensor matrix_batch) { + const int batch_size = matrix_batch.dimension(0); + const int m = matrix_batch.dimension(1); + const int n = matrix_batch.dimension(2); + CudaLaunchConfig config = GetCudaLaunchConfig(batch_size * m * n, device); + EyeKernel<<>>(config.virtual_thread_count, batch_size, m, + n, matrix_batch.data()); + } +}; + +template struct EyeFunctor; +template struct EyeFunctor; +template struct EyeFunctor; +template struct EyeFunctor; + +} // namespace functor +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/function_ops.cc b/tensorflow/core/kernels/function_ops.cc index 1c6026c25db262373fdacd951357bf68f5dde799..f2290e87a5fdac44629ed6b81c8661cf74c2054e 100644 --- a/tensorflow/core/kernels/function_ops.cc +++ b/tensorflow/core/kernels/function_ops.cc @@ -328,9 +328,10 @@ class RemoteCallOp : public AsyncOpKernel { lib->Run(opts, handle, args, rets, [rets, done, ctx](const Status& status) { if (!status.ok()) { ctx->SetStatus(status); - } - for (size_t i = 0; i < rets->size(); ++i) { - ctx->set_output(i, (*rets)[i]); + } else { + for (size_t i = 0; i < rets->size(); ++i) { + ctx->set_output(i, (*rets)[i]); + } } delete rets; done(); diff --git a/tensorflow/core/kernels/gather_functor.cc b/tensorflow/core/kernels/gather_functor.cc index 1b8be9b2cea879ca992bf484f247bb830ac6ba1f..dde08b37eacb9edada92f98c5115f694015aad34 100644 --- a/tensorflow/core/kernels/gather_functor.cc +++ b/tensorflow/core/kernels/gather_functor.cc @@ -28,7 +28,7 @@ namespace functor { #define DECLARE_GPU_SPECS_INDEX(T, Index) \ template <> \ int64 GatherFunctor::operator()( \ - const GPUDevice& d, typename TTypes::ConstTensor Tparams, \ + OpKernelContext* ctx, typename TTypes::ConstTensor Tparams, \ typename TTypes::ConstFlat Tindices, \ typename TTypes::Tensor Tout); \ extern template struct GatherFunctor; diff --git a/tensorflow/core/kernels/gather_functor.h b/tensorflow/core/kernels/gather_functor.h index dfa1a5f1f90b498bf28ace363437fe4ea4e51ce9..1e429a037e8b16f5e01766125e1d10ec7567d78d 100644 --- a/tensorflow/core/kernels/gather_functor.h +++ b/tensorflow/core/kernels/gather_functor.h @@ -23,6 +23,8 @@ limitations under the License. #include "tensorflow/core/kernels/bounds_check.h" #include "tensorflow/core/platform/prefetch.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/util/work_sharder.h" namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; @@ -32,7 +34,8 @@ namespace functor { // Helper method to copy using memcpy. template -SliceIndex HandleCopies(typename TTypes::ConstTensor params, +SliceIndex HandleCopies(OpKernelContext* ctx, + typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, SliceIndex slice_elems, typename TTypes::Tensor out) { @@ -47,44 +50,64 @@ SliceIndex HandleCopies(typename TTypes::ConstTensor params, } // Compute slice_bytes here so that static knowledge is available const size_t slice_bytes = slice_elems * sizeof(T); - for (SliceIndex b = 0; b < batch_size; b++) { - for (SliceIndex i = 0; i < indices_size; i++) { - const SliceIndex i_next = i + 1; - const SliceIndex b_next = b + 1; - if (i_next < indices_size) { - port::prefetch(¶ms(b, indices(i_next), 0)); - port::prefetch(&out(b, i_next, 0)); - } else if (b_next < batch_size) { + auto worker_threads = ctx->device()->tensorflow_cpu_worker_threads(); + mutex mu; + // Store the value of invalidate index for printing error information, it's a shared variable. + SliceIndex result = -1; + auto work = [&] (int64 start, int64 end) { + SliceIndex batch_idx = static_cast(start / indices_size); + SliceIndex indices_idx = static_cast(start % indices_size); + SliceIndex batch_idx_end = static_cast(end / indices_size); + SliceIndex indices_idx_end = static_cast(end % indices_size); + + while ((batch_idx < batch_idx_end) || + (batch_idx == batch_idx_end && indices_idx < indices_idx_end)) { + SliceIndex i_next = indices_idx + 1; + SliceIndex b_next = batch_idx + 1; + if ((batch_idx == batch_idx_end && i_next < indices_idx_end) || + (i_next < indices_size)) { + port::prefetch(¶ms(batch_idx, indices(i_next), 0)); + port::prefetch(&out(batch_idx, i_next, 0)); + b_next = batch_idx; + } else if (b_next <= batch_idx_end) { port::prefetch(¶ms(b_next, indices(0), 0)); port::prefetch(&out(b_next, 0, 0)); + i_next = 0; + } + const Index index = internal::SubtleMustCopy(indices(indices_idx)); + if (!FastBoundsCheck(index, limit)) { + mutex_lock l(mu); + result = indices_idx; + return; } - // Grab the index and check its validity. An earlier version of the - // code checked it and then grabbed it from memory a second time, which - // was a security risk since it could have changed in between. - const Index index = internal::SubtleMustCopy(indices(i)); - if (!FastBoundsCheck(index, limit)) return i; // Copy using memcpy if possible, otherwise an Eigen loop // TODO(cwhipkey): avoid linking to framework to get Allocator (to improve // ahead-of-time compilation binary size). if (is_simple_type::value) { // Avoid auto-promotion to Index from SliceIndex by casting. - memcpy(out_base + (b * indices_size + i) * slice_elems, - params_base + (b * static_cast(limit) + + memcpy(out_base + (batch_idx * indices_size + indices_idx) * slice_elems, + params_base + (batch_idx * static_cast(limit) + static_cast(index)) * - slice_elems, + slice_elems, slice_bytes); } else { // For non-"simple" types (e.g. strings). - out.template chip<1>(i) = params.template chip<1>(index); + out.template chip<1>(indices_idx) = params.template chip<1>(index); } + indices_idx = i_next; + batch_idx = b_next; } - } - return -1; + }; + + Shard(worker_threads->num_threads, worker_threads->workers, batch_size*indices_size, + slice_elems * sizeof(T), work); + return result; } template struct GatherFunctorCPU { - int64 operator()(typename TTypes::ConstTensor params, + int64 operator()(OpKernelContext* ctx, + typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out) { const int64 N = indices.size(); @@ -94,16 +117,16 @@ struct GatherFunctorCPU { bool use_large = (slice_size > std::numeric_limits::max() || params.size() > std::numeric_limits::max() || N > std::numeric_limits::max()); -#define CALL(elems) \ - do { \ - if (use_large) { \ - bad_i = HandleCopies(params, indices, \ - slice_size, out); \ - } else { \ - const int32 small_slice = static_cast(slice_size); \ - bad_i = HandleCopies(params, indices, \ - small_slice, out); \ - } \ +#define CALL(elems) \ + do { \ + if (use_large) { \ + bad_i = HandleCopies(ctx, params, indices, \ + slice_size, out); \ + } else { \ + const int32 small_slice = static_cast(slice_size); \ + bad_i = HandleCopies(ctx, params, indices, \ + small_slice, out); \ + } \ } while (0) if (slice_size == 10) @@ -120,18 +143,18 @@ struct GatherFunctorCPU { template struct GatherFunctor { - int64 operator()(const Device& d, typename TTypes::ConstTensor params, + int64 operator()(OpKernelContext* ctx, typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out); }; template struct GatherFunctor { - int64 operator()(const CPUDevice& d, + int64 operator()(OpKernelContext* ctx, typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out) { - return GatherFunctorCPU()(params, indices, out); + return GatherFunctorCPU()(ctx, params, indices, out); } }; diff --git a/tensorflow/core/kernels/gather_functor_gpu.cu.h b/tensorflow/core/kernels/gather_functor_gpu.cu.h index e2384ef01151e3c2c31b9607542f6f5cc4e9588d..a50b51b54b1d8e23b4082ba7b6bee8db2cc28382 100644 --- a/tensorflow/core/kernels/gather_functor_gpu.cu.h +++ b/tensorflow/core/kernels/gather_functor_gpu.cu.h @@ -72,10 +72,11 @@ __global__ void GatherOpKernel(const T* params, const Index* indices, T* out, namespace functor { template struct GatherFunctor { - int64 operator()(const GPUDevice& d, + int64 operator()(OpKernelContext* ctx, typename TTypes::ConstTensor params, typename TTypes::ConstFlat indices, typename TTypes::Tensor out) { + const GPUDevice& d = ctx->eigen_gpu_device(); const int64 out_size = out.size(); if (out_size == 0) { // We need a check here since the CPU version does useful error checking diff --git a/tensorflow/core/kernels/gather_op.cc b/tensorflow/core/kernels/gather_op.cc index e649c54fa80a7b3f1c18a1ac6fa453105580cdf8..239d5d2e990a88bbc8ca5949a07a2aa2a75de2ba 100644 --- a/tensorflow/core/kernels/gather_op.cc +++ b/tensorflow/core/kernels/gather_op.cc @@ -106,7 +106,7 @@ class GatherOp : public OpKernel { auto out_flat = out->shaped({outer_size, N, inner_size}); functor::GatherFunctor functor; - int64 bad_i = functor(c->eigen_device(), params_flat, + int64 bad_i = functor(c, params_flat, indices_flat, out_flat); OP_REQUIRES( @@ -140,6 +140,8 @@ class GatherOp : public OpKernel { // Registration of the CPU implementations. TF_CALL_ALL_TYPES(REGISTER_GATHER_CPU); TF_CALL_QUANTIZED_TYPES(REGISTER_GATHER_CPU); +TF_CALL_quint16(REGISTER_GATHER_CPU); +TF_CALL_qint16(REGISTER_GATHER_CPU); #undef REGISTER_GATHER_CPU diff --git a/tensorflow/core/kernels/histogram_op.cc b/tensorflow/core/kernels/histogram_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..4e035286f6f15454b9065c54888b59ef624827db --- /dev/null +++ b/tensorflow/core/kernels/histogram_op.cc @@ -0,0 +1,147 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/math_ops.cc. + +#define EIGEN_USE_THREADS + +#include "tensorflow/core/kernels/histogram_op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef Eigen::GpuDevice GPUDevice; + +namespace functor { + +template +struct HistogramFixedWidthFunctor { + static Status Compute(OpKernelContext* context, + const typename TTypes::ConstTensor& values, + const typename TTypes::ConstTensor& value_range, + int32 nbins, typename TTypes::Tensor& out) { + const CPUDevice& d = context->eigen_device(); + + Tensor index_to_bin_tensor; + + TF_RETURN_IF_ERROR(context->forward_input_or_allocate_temp( + {0}, DataTypeToEnum::value, TensorShape({values.size()}), + &index_to_bin_tensor)); + auto index_to_bin = index_to_bin_tensor.flat(); + + const double step = static_cast(value_range(1) - value_range(0)) / + static_cast(nbins); + + // The calculation is done by finding the slot of each value in `values`. + // With [a, b]: + // step = (b - a) / nbins + // (x - a) / step + // , then the entries are mapped to output. + index_to_bin.device(d) = + ((values.cwiseMax(value_range(0)) - values.constant(value_range(0))) + .template cast() / + step) + .template cast() + .cwiseMin(nbins - 1); + + out.setZero(); + for (int32 i = 0; i < index_to_bin.size(); i++) { + out(index_to_bin(i)) += Tout(1); + } + return Status::OK(); + } +}; + +} // namespace functor + +template +class HistogramFixedWidthOp : public OpKernel { + public: + explicit HistogramFixedWidthOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + const Tensor& values_tensor = ctx->input(0); + const Tensor& value_range_tensor = ctx->input(1); + const Tensor& nbins_tensor = ctx->input(2); + + OP_REQUIRES(ctx, TensorShapeUtils::IsVector(value_range_tensor.shape()), + errors::InvalidArgument("value_range should be a vector.")); + OP_REQUIRES(ctx, (value_range_tensor.shape().num_elements() == 2), + errors::InvalidArgument( + "value_range should be a vector of 2 elements.")); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(nbins_tensor.shape()), + errors::InvalidArgument("nbins should be a scalar.")); + + const auto values = values_tensor.flat(); + const auto value_range = value_range_tensor.flat(); + const auto nbins = nbins_tensor.scalar()(); + + OP_REQUIRES( + ctx, (value_range(0) < value_range(1)), + errors::InvalidArgument("value_range should satisfy value_range[0] < " + "value_range[1], but got '[", + value_range(0), ", ", value_range(1), "]'")); + OP_REQUIRES( + ctx, (nbins > 0), + errors::InvalidArgument("nbins should be a positive number, but got '", + nbins, "'")); + + Tensor* out_tensor; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, TensorShape({nbins}), &out_tensor)); + auto out = out_tensor->flat(); + + OP_REQUIRES_OK( + ctx, functor::HistogramFixedWidthFunctor::Compute( + ctx, values, value_range, nbins, out)); + } +}; + +#define REGISTER_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("dtype"), \ + HistogramFixedWidthOp) \ + REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("dtype"), \ + HistogramFixedWidthOp) + +TF_CALL_REAL_NUMBER_TYPES(REGISTER_KERNELS); +#undef REGISTER_KERNELS + +#if GOOGLE_CUDA +#define REGISTER_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("HistogramFixedWidth") \ + .Device(DEVICE_GPU) \ + .HostMemory("value_range") \ + .HostMemory("nbins") \ + .TypeConstraint("T") \ + .TypeConstraint("dtype"), \ + HistogramFixedWidthOp) + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_KERNELS); +#undef REGISTER_KERNELS + +#endif // GOOGLE_CUDA + +} // end namespace tensorflow diff --git a/tensorflow/core/kernels/histogram_op.h b/tensorflow/core/kernels/histogram_op.h new file mode 100644 index 0000000000000000000000000000000000000000..1b253f7fed5b09ce7d93362e2465951ba969922a --- /dev/null +++ b/tensorflow/core/kernels/histogram_op.h @@ -0,0 +1,38 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_HISTOGRAM_OP_H_ +#define TENSORFLOW_HISTOGRAM_OP_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" + +namespace tensorflow { +namespace functor { + +template +struct HistogramFixedWidthFunctor { + static Status Compute(OpKernelContext* context, + const typename TTypes::ConstTensor& values, + const typename TTypes::ConstTensor& value_range, + int32 nbins, typename TTypes::Tensor& out); +}; + +} // end namespace functor +} // end namespace tensorflow + +#endif // TENSORFLOW_HISTOGRAM_OP_H_ diff --git a/tensorflow/core/kernels/histogram_op_gpu.cu.cc b/tensorflow/core/kernels/histogram_op_gpu.cu.cc new file mode 100644 index 0000000000000000000000000000000000000000..c2bb958be8b29c4a6df99cf5533748d7db73179c --- /dev/null +++ b/tensorflow/core/kernels/histogram_op_gpu.cu.cc @@ -0,0 +1,125 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA + +#define EIGEN_USE_GPU + +#include "tensorflow/core/kernels/histogram_op.h" +#include "external/cub_archive/cub/device/device_histogram.cuh" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_kernel_helper.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + +namespace tensorflow { + +typedef Eigen::GpuDevice GPUDevice; + +namespace functor { + +// TODO(yongtang) int64 of atomicAdd is not supported yet. +template +struct HistogramFixedWidthFunctor { + static Status Compute(OpKernelContext* context, + const typename TTypes::ConstTensor& values, + const typename TTypes::ConstTensor& value_range, + int32 nbins, typename TTypes::Tensor& out) { + tensorflow::AllocatorAttributes pinned_allocator; + pinned_allocator.set_on_host(true); + pinned_allocator.set_gpu_compatible(true); + + Tensor levels_tensor; + TF_RETURN_IF_ERROR(context->allocate_temp( + DataTypeToEnum::value, TensorShape({nbins + 1}), &levels_tensor, + pinned_allocator)); + auto levels = levels_tensor.flat(); + + const double step = static_cast(value_range(1) - value_range(0)) / + static_cast(nbins); + levels(0) = std::numeric_limits::lowest(); + for (int i = 1; i < nbins; i++) { + levels(i) = + static_cast(static_cast(value_range(0)) + step * i); + } + levels(nbins) = std::numeric_limits::max(); + + size_t temp_storage_bytes = 0; + const T* d_samples = values.data(); + Tout* d_histogram = out.data(); + int num_levels = levels.size(); + T* d_levels = levels.data(); + int num_samples = values.size(); + const cudaStream_t& stream = GetCudaStream(context); + + // The first HistogramRange is to obtain the temp storage size required + // with d_temp_storage = NULL passed to the call. + auto err = cub::DeviceHistogram::HistogramRange( + /* d_temp_storage */ NULL, + /* temp_storage_bytes */ temp_storage_bytes, + /* d_samples */ d_samples, + /* d_histogram */ d_histogram, + /* num_levels */ num_levels, + /* d_levels */ d_levels, + /* num_samples */ num_samples, + /* stream */ stream); + if (err != cudaSuccess) { + return errors::Internal( + "Could not launch HistogramRange to get temp storage: ", + cudaGetErrorString(err), "."); + } + + Tensor temp_storage; + TF_RETURN_IF_ERROR(context->allocate_temp( + DataTypeToEnum::value, + TensorShape({static_cast(temp_storage_bytes)}), &temp_storage)); + + void* d_temp_storage = temp_storage.flat().data(); + + // The second HistogramRange is to actual run with d_temp_storage + // allocated with temp_storage_bytes. + err = cub::DeviceHistogram::HistogramRange( + /* d_temp_storage */ d_temp_storage, + /* temp_storage_bytes */ temp_storage_bytes, + /* d_samples */ d_samples, + /* d_histogram */ d_histogram, + /* num_levels */ num_levels, + /* d_levels */ d_levels, + /* num_samples */ num_samples, + /* stream */ stream); + if (err != cudaSuccess) { + return errors::Internal("Could not launch HistogramRange: ", + cudaGetErrorString(err), "."); + } + + return Status::OK(); + } +}; + +} // end namespace functor + +#define REGISTER_GPU_SPEC(type) \ + template struct functor::HistogramFixedWidthFunctor; + +TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_SPEC); +#undef REGISTER_GPU_SPEC + +} // namespace tensorflow + +#endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/iterator_ops.cc b/tensorflow/core/kernels/iterator_ops.cc index df13edc83aed69954263c1922767766dc80be9d0..d8bcd09842c51f30e499de7c3a2d58c08036a202 100644 --- a/tensorflow/core/kernels/iterator_ops.cc +++ b/tensorflow/core/kernels/iterator_ops.cc @@ -16,9 +16,11 @@ limitations under the License. #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/common_runtime/graph_runner.h" +#include "tensorflow/core/framework/iterator.pb.h" #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/resource_op_kernel.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/threadpool.h" @@ -35,6 +37,8 @@ namespace { // See documentation in ../ops/dataset_ops.cc for a high-level // description of the following ops. +const char kIteratorVariantTypeName[] = "tensorflow::Iterator"; + Status VerifyTypesMatch(const DataTypeVector& expected, const DataTypeVector& received) { if (expected.size() != received.size()) { @@ -93,10 +97,10 @@ class IteratorResource : public ResourceBase { } } - Status Save(OpKernelContext* ctx, const string& path) { + Status Save(IteratorStateWriter* writer) { std::shared_ptr captured_iterator(iterator_); if (captured_iterator) { - return captured_iterator->Save(ctx, path); + return captured_iterator->Save(writer); } else { return errors::FailedPrecondition( "Save() failed because the iterator has not been initialized. " @@ -105,49 +109,34 @@ class IteratorResource : public ResourceBase { } } - Status Restore(OpKernelContext* ctx, const string& path) { - if (!(ctx->env()->FileExists(MetaFilename(path)).ok())) { - return errors::NotFound( - "Failed to restore Iterator state. No file found at ", - MetaFilename(path)); - } - - BundleReader bundle_reader(ctx->env(), path); - TF_RETURN_IF_ERROR(bundle_reader.status()); - BundleReaderWrapper reader(&bundle_reader); - if (reader.Contains(GraphDatasetBase::kDatasetGraphKey)) { - string serialized_graph_def; - TF_RETURN_IF_ERROR(reader.ReadScalar(GraphDatasetBase::kDatasetGraphKey, - &serialized_graph_def)); - GraphDef graph_def; - graph_def.ParseFromString(serialized_graph_def); - // TODO(srbs): Is there a way of getting the op registry of the original - // graph. - Graph graph(OpRegistry::Global()); - TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr)); - string output_node; - TF_RETURN_IF_ERROR(reader.ReadScalar( - GraphDatasetBase::kDatasetGraphOutputNodeKey, &output_node)); - std::vector outputs; - GraphRunner graph_runner(ctx->env()); - TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {}, - {output_node}, &outputs)); - DatasetBase* dataset; - TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset)); - TF_RETURN_IF_ERROR(set_iterator(dataset->MakeIterator("Iterator"))); - } else if (reader.Contains(IteratorBase::kIteratorExhausted)) { - TF_RETURN_IF_ERROR(set_iterator(std::unique_ptr( - new ExhaustedIterator(output_dtypes_, output_shapes_)))); + Status Restore(OpKernelContext* ctx, IteratorStateReader* reader) { + string serialized_graph_def; + TF_RETURN_IF_ERROR(reader->ReadScalar(GraphDatasetBase::kDatasetGraphKey, + &serialized_graph_def)); + GraphDef graph_def; + if (!graph_def.ParseFromString(serialized_graph_def)) { + return errors::Internal("Error parsing dataset GraphDef."); } + string output_node; + TF_RETURN_IF_ERROR(reader->ReadScalar( + GraphDatasetBase::kDatasetGraphOutputNodeKey, &output_node)); + DatasetBase* dataset = nullptr; + Graph graph(OpRegistry::Global()); + TF_RETURN_IF_ERROR(ImportGraphDef({}, graph_def, &graph, nullptr)); + std::vector outputs; + GraphRunner graph_runner(ctx->env()); + TF_RETURN_IF_ERROR(graph_runner.Run(&graph, ctx->function_library(), {}, + {output_node}, &outputs)); + TF_RETURN_IF_ERROR(GetDatasetFromVariantTensor(outputs[0], &dataset)); + + TF_RETURN_IF_ERROR(set_iterator(dataset->MakeIterator("Iterator"))); std::shared_ptr captured_iterator(iterator_); if (captured_iterator) { - // TODO(srbs): Figure a way to pass bundle_reader here. - return captured_iterator->Restore(ctx, path); + return captured_iterator->Restore(ctx, reader); } else { return errors::FailedPrecondition( - "Failed to restore iterator from ", path, - ". Make sure the checkpoint ", + "Failed to restore iterator. Make sure the checkpoint ", "is not corrupt. If the checkpoint does not contain the GraphDef, ", "you will need to initialize your iterator before restoring."); } @@ -174,43 +163,194 @@ class IteratorResource : public ResourceBase { } private: - // A no-op iterator which always sets end_of_sequence = true. An instance of - // this is returned when attempting to restore an exhausted iterator. This is - // needed because the Dataset GraphDef may not have been saved for exhausted - // iterators so the actual Iterator can not be built. - class ExhaustedIterator : public IteratorBase { - public: - ExhaustedIterator(const DataTypeVector& output_dtypes, - const std::vector& output_shapes) - : output_dtypes_(output_dtypes), output_shapes_(output_shapes) {} - Status GetNext(IteratorContext* ctx, std::vector* out_tensors, - bool* end_of_sequence) final { - *end_of_sequence = true; - return Status::OK(); - } + std::shared_ptr iterator_; + const DataTypeVector output_dtypes_; + const std::vector output_shapes_; +}; + +// Helper class for reading data from a VariantTensorData object. +class VariantTensorDataReader : public IteratorStateReader { + public: + explicit VariantTensorDataReader(const VariantTensorData* data) + : data_(data) { + PreProcess(); + } + + // Returns OK iff the initialization was successful, i.e., + // pre-processing did not have errors. + Status status() const { return status_; } + + Status ReadScalar(StringPiece key, int64* val) override { + return ReadScalarInternal(key, val); + } + + Status ReadScalar(StringPiece key, string* val) override { + return ReadScalarInternal(key, val); + } - const DataTypeVector& output_dtypes() const override { - return output_dtypes_; + bool Contains(StringPiece key) override { + return map_.find(key.ToString()) != map_.end(); + } + + private: + void PreProcess() { + string metadata; + data_->get_metadata(&metadata); + IteratorStateMetadata proto; + if (!proto.ParseFromString(metadata)) { + status_ = errors::Internal("Error parsing IteratorStateMetadata."); + return; + } + size_t num_entries = proto.keys_size(); + CHECK_EQ(num_entries, data_->tensors_size()); + for (size_t i = 0; i < num_entries; i++) { + map_[proto.keys(i)] = i; } + } - const std::vector& output_shapes() const override { - return output_shapes_; + template + Status ReadScalarInternal(StringPiece key, T* val) { + if (map_.find(key.ToString()) == map_.end()) { + return errors::NotFound(key); } + *val = data_->tensors(map_[key.ToString()]).scalar()(); + return Status::OK(); + } - virtual const std::vector& output_shapes() { - return output_shapes_; + std::map map_; + const VariantTensorData* data_; // Not owned. + Status status_; +}; + +// Helper class for writing data to a VariantTensorData object. +class VariantTensorDataWriter : public IteratorStateWriter { + public: + // Does not take ownership of data. + explicit VariantTensorDataWriter(VariantTensorData* data) : data_(data) {} + + Status WriteScalar(StringPiece key, const int64 val) override { + return WriteScalarInternal(key, val); + } + + Status WriteScalar(StringPiece key, const string& val) override { + return WriteScalarInternal(key, val); + } + + // Writes the metadata to `data_`. + Status Flush() { + string metadata; + if (!metadata_proto_.SerializeToString(&metadata)) { + return errors::Internal("Unable to serialize IteratorStateMetadata."); } + data_->set_metadata(metadata); + return Status::OK(); + } - private: - const DataTypeVector output_dtypes_; - const std::vector output_shapes_; - }; + private: + template + Status WriteScalarInternal(StringPiece key, const T& val) { + // Write key to the metadata proto. This gets written to `data_` + // when `Flush()` is called. We do this lazily to avoid multiple + // serialization calls. + metadata_proto_.add_keys(key.ToString()); + + // Update tensors. + Tensor val_t = Tensor(DataTypeToEnum::v(), TensorShape({})); + val_t.scalar()() = val; + *(data_->add_tensors()) = std::move(val_t); + return Status::OK(); + } - std::shared_ptr iterator_; - const DataTypeVector output_dtypes_; - const std::vector output_shapes_; + VariantTensorData* data_; + // TODO(srbs): Set the version string. + IteratorStateMetadata metadata_proto_; +}; + +// Wrapper for encoding/decoding the iterator state stored in a Variant tensor. +// The get() method returns an IteratorStateReader which can be used +// to restore iterator state. +// +// Usage example: +// +// Encoding: +// +// Tensor t(DT_VARIANT, TensorShape({})); +// t->scalar()() = IteratorStateVariant(iterator_resource); +// +// Encode() sets the type_name of the VariantTensorData object to +// IteratorStateVariant::TypeName(). +// +// Decoding: +// +// Variant v = ; +// DecodeUnaryVariant(&v); +// IteratorStateVariant* wrapper = v.get(); +// iterator_resource->Restore(ctx, wrapper->get()) +// +// The type_name of the VariantTensorData object to be decoded must +// match IteratorStateVariant::TypeName(). +class IteratorStateVariant { + public: + IteratorStateVariant() : data_(nullptr) {} + IteratorStateVariant(const IteratorStateVariant& other) : data_(nullptr) { + if (other.data_) { + Decode(*other.data_); + } + } + // Initializes this object with the current state of the iterator so + // that it can be written on the next call to Encode(). + Status InitializeFromIterator(IteratorResource* iterator_resource) { + data_.reset(new VariantTensorData()); + data_->set_type_name(TypeName()); + VariantTensorDataWriter writer(data_.get()); + TF_RETURN_IF_ERROR(iterator_resource->Save(&writer)); + TF_RETURN_IF_ERROR(writer.Flush()); + return Status::OK(); + } + string TypeName() const { return kIteratorVariantTypeName; } + void Encode(VariantTensorData* data) const { *data = *data_; } + bool Decode(const VariantTensorData& data) { + if (data.type_name() != TypeName()) { + return false; + } + std::unique_ptr tensor_data(new VariantTensorData); + *tensor_data = data; + std::unique_ptr reader( + new VariantTensorDataReader(tensor_data.get())); + status_ = reader->status(); + if (!status_.ok()) { + return false; + } + data_ = std::move(tensor_data); + reader_ = std::move(reader); + return true; + } + IteratorStateReader* get() { return reader_.get(); } + Status status() const { return status_; } + string DebugString() const { + if (data_) { + return strings::StrCat("IteratorStateVariant<", + "data: ", data_->DebugString(), + " status: ", status_.ToString(), ">"); + } else { + return strings::StrCat("IteratorStateVariant"); + } + } + + private: + std::unique_ptr reader_; + Status status_; + std::unique_ptr data_; }; +// Register the reader class in the global variant decode_fn registry +// so that a Variant containing a serialized representation of iterator state +// can be decoded using DecodeUnaryVariant. If we don't do this we will need +// to manually decode the returned Variant using MaybeDecodeAndCopy in +// DeserializeIteratorOp which is not recommended. +REGISTER_UNARY_VARIANT_DECODE_FUNCTION(IteratorStateVariant, + kIteratorVariantTypeName); + // TODO(mrry): Can we simply use the template kernel here? class IteratorHandleOp : public ResourceOpKernel { public: @@ -294,37 +434,6 @@ class ToSingleElementOp : public OpKernel { } }; -class SaveIteratorOp : public OpKernel { - public: - explicit SaveIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - - void Compute(OpKernelContext* ctx) override { - IteratorResource* iterator_resource; - OP_REQUIRES_OK( - ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); - OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->input(1).shape()), - errors::InvalidArgument("SaveIteratorOp: path must be scalar")); - const string& path = ctx->input(1).scalar()(); - OP_REQUIRES_OK(ctx, iterator_resource->Save(ctx, path)); - } -}; - -class RestoreIteratorOp : public OpKernel { - public: - explicit RestoreIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - - void Compute(OpKernelContext* ctx) override { - IteratorResource* iterator_resource; - OP_REQUIRES_OK( - ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); - OP_REQUIRES( - ctx, TensorShapeUtils::IsScalar(ctx->input(1).shape()), - errors::InvalidArgument("RestoreIteratorOp: path must be scalar")); - const string& path = ctx->input(1).scalar()(); - OP_REQUIRES_OK(ctx, iterator_resource->Restore(ctx, path)); - } -}; - class OneShotIteratorOp : public AsyncOpKernel { public: explicit OneShotIteratorOp(OpKernelConstruction* ctx) @@ -644,15 +753,55 @@ class IteratorFromStringHandleOp : public OpKernel { std::vector output_shapes_; }; +class SerializeIteratorOp : public OpKernel { + public: + explicit SerializeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + const Tensor& resource_handle_t = ctx->input(0); + OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(resource_handle_t.shape()), + errors::InvalidArgument("resource_handle must be a scalar")); + + // Validate that the handle corresponds to a real resource, and + // that it is an IteratorResource. + IteratorResource* iterator_resource; + OP_REQUIRES_OK( + ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); + iterator_resource->Unref(); + Tensor* variant_t; + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape({}), &variant_t)); + IteratorStateVariant v; + OP_REQUIRES_OK(ctx, v.InitializeFromIterator(iterator_resource)); + variant_t->scalar()() = v; + } +}; + +class DeserializeIteratorOp : public OpKernel { + public: + explicit DeserializeIteratorOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + + void Compute(OpKernelContext* ctx) override { + // Validate that the handle corresponds to a real resource, and + // that it is an IteratorResource. + IteratorResource* iterator_resource; + OP_REQUIRES_OK( + ctx, LookupResource(ctx, HandleFromInput(ctx, 0), &iterator_resource)); + + Variant variant = ctx->input(1).scalar()(); + auto* wrapper = variant.get(); + OP_REQUIRES(ctx, wrapper != nullptr, + errors::InvalidArgument( + "DeserializeIteratorOp: Unable to parse variant tensor.")); + OP_REQUIRES_OK(ctx, wrapper->status()); + OP_REQUIRES_OK(ctx, iterator_resource->Restore(ctx, wrapper->get())); + } +}; + REGISTER_KERNEL_BUILDER(Name("Iterator").Device(DEVICE_CPU), IteratorHandleOp); REGISTER_KERNEL_BUILDER(Name("MakeIterator").Device(DEVICE_CPU), MakeIteratorOp); REGISTER_KERNEL_BUILDER(Name("DatasetToSingleElement").Device(DEVICE_CPU), ToSingleElementOp); -REGISTER_KERNEL_BUILDER(Name("SaveIterator").Device(DEVICE_CPU), - SaveIteratorOp); -REGISTER_KERNEL_BUILDER(Name("RestoreIterator").Device(DEVICE_CPU), - RestoreIteratorOp); REGISTER_KERNEL_BUILDER(Name("OneShotIterator").Device(DEVICE_CPU), OneShotIteratorOp); REGISTER_KERNEL_BUILDER(Name("IteratorGetNext").Device(DEVICE_CPU), @@ -661,6 +810,10 @@ REGISTER_KERNEL_BUILDER(Name("IteratorToStringHandle").Device(DEVICE_CPU), IteratorToStringHandleOp); REGISTER_KERNEL_BUILDER(Name("IteratorFromStringHandle").Device(DEVICE_CPU), IteratorFromStringHandleOp); +REGISTER_KERNEL_BUILDER(Name("SerializeIterator").Device(DEVICE_CPU), + SerializeIteratorOp); +REGISTER_KERNEL_BUILDER(Name("DeserializeIterator").Device(DEVICE_CPU), + DeserializeIteratorOp); } // namespace diff --git a/tensorflow/core/kernels/linalg_ops_common.h b/tensorflow/core/kernels/linalg_ops_common.h index 1d31786728f5c4aac023d7c4ef1e347577267110..f7c3f1950b9af31769132e4792adc6718682bf28 100644 --- a/tensorflow/core/kernels/linalg_ops_common.h +++ b/tensorflow/core/kernels/linalg_ops_common.h @@ -172,13 +172,14 @@ extern template class LinearAlgebraOp; } // namespace tensorflow -#define INHERIT_LINALG_TYPEDEFS(Scalar) \ - typedef LinearAlgebraOp Base; \ - using Matrix = typename Base::Matrix; \ - using MatrixMap = typename Base::MatrixMap; \ - using MatrixMaps = typename Base::MatrixMaps; \ - using ConstMatrixMap = typename Base::ConstMatrixMap; \ - using ConstMatrixMaps = typename Base::ConstMatrixMaps; \ +#define INHERIT_LINALG_TYPEDEFS(Scalar) \ + typedef LinearAlgebraOp Base; \ + using RealScalar = typename Eigen::NumTraits::Real; \ + using Matrix = typename Base::Matrix; \ + using MatrixMap = typename Base::MatrixMap; \ + using MatrixMaps = typename Base::MatrixMaps; \ + using ConstMatrixMap = typename Base::ConstMatrixMap; \ + using ConstMatrixMaps = typename Base::ConstMatrixMaps; \ using TensorShapes = typename Base::TensorShapes; #define REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) \ diff --git a/tensorflow/core/kernels/listdiff_op.cc b/tensorflow/core/kernels/listdiff_op.cc index d303bdd5605336e04602d40111af42e2e062dcc2..d28a2729d4c88d8603ade38d1c46531ca2bef47b 100644 --- a/tensorflow/core/kernels/listdiff_op.cc +++ b/tensorflow/core/kernels/listdiff_op.cc @@ -24,12 +24,13 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" namespace tensorflow { -template +template class ListDiffOp : public OpKernel { public: explicit ListDiffOp(OpKernelConstruction* context) : OpKernel(context) { const DataType dt = DataTypeToEnum::v(); - OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, DT_INT32})); + const DataType dtidx = DataTypeToEnum::v(); + OP_REQUIRES_OK(context, context->MatchSignature({dt, dt}, {dt, dtidx})); } void Compute(OpKernelContext* context) override { @@ -72,9 +73,9 @@ class ListDiffOp : public OpKernel { Tensor* indices = nullptr; OP_REQUIRES_OK(context, context->allocate_output(1, {out_size}, &indices)); - auto Tindices = indices->vec(); + auto Tindices = indices->vec(); - for (int i = 0, p = 0; i < static_cast(x_size); ++i) { + for (Tidx i = 0, p = 0; i < static_cast(x_size); ++i) { if (y_set.count(Tx(i)) == 0) { OP_REQUIRES(context, p < out_size, errors::InvalidArgument( @@ -95,7 +96,12 @@ class ListDiffOp : public OpKernel { .Device(DEVICE_CPU) \ .TypeConstraint("T") \ .TypeConstraint("out_idx"), \ - ListDiffOp) + ListDiffOp) \ + REGISTER_KERNEL_BUILDER(Name("ListDiff") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("out_idx"), \ + ListDiffOp) TF_CALL_REAL_NUMBER_TYPES(REGISTER_LISTDIFF); REGISTER_LISTDIFF(string); diff --git a/tensorflow/core/kernels/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/map_and_batch_dataset_op.cc index 332a96ae032c5797d34bc4430ca996bbcd41c27c..620efdb7781e677c94af4946033e02955ee412f3 100644 --- a/tensorflow/core/kernels/map_and_batch_dataset_op.cc +++ b/tensorflow/core/kernels/map_and_batch_dataset_op.cc @@ -287,10 +287,12 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { EXCLUSIVE_LOCKS_REQUIRED(mu_) { port::Tracing::TraceMe activity(strings::StrCat(prefix(), "::Start")); // Initialize batch result. - mutex_lock l(batch_results_[batch_index].mu); - batch_results_[batch_index].output_allocated = false; - batch_results_[batch_index].counter.reset( - new BlockingCounter(dataset()->batch_size_)); + { + mutex_lock l(batch_results_[batch_index].mu); + batch_results_[batch_index].output_allocated = false; + batch_results_[batch_index].counter.reset( + new BlockingCounter(dataset()->batch_size_)); + } // Initialize invocation results. for (size_t i = 0; i < dataset()->batch_size_; ++i) { size_t index = ComputeInvocationIndex(batch_index, i); @@ -334,7 +336,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel { const DataTypeVector output_types_; const std::vector output_shapes_; const std::unique_ptr captured_func_; - const Eigen::ThreadPoolDevice* device_; // not owned + const Eigen::ThreadPoolDevice* device_; // not owned }; const int graph_def_version_; diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/matrix_inverse_op.cc index 832e508bb70e4fc6ebf6678229413c0b82494ad4..c61a091c7b7b82dbcb6e7b7f016e9cd2361f3f51 100644 --- a/tensorflow/core/kernels/matrix_inverse_op.cc +++ b/tensorflow/core/kernels/matrix_inverse_op.cc @@ -33,6 +33,8 @@ limitations under the License. #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/kernels/eye_functor.h" +#include "tensorflow/core/kernels/transpose_functor.h" #endif namespace tensorflow { @@ -68,7 +70,6 @@ class MatrixInverseOp : public LinearAlgebraOp { // a result of basic user mistakes, such as providing integer valued // matrices that are exactly singular, or due to underflow if this // code is run with denormals being flushed to zero. - using RealScalar = typename Base::RealScalar; const RealScalar min_abs_pivot = lu_decomposition.matrixLU().diagonal().cwiseAbs().minCoeff(); OP_REQUIRES(context, min_abs_pivot > RealScalar(0), @@ -135,15 +136,15 @@ class MatrixInverseOpGpu : public AsyncOpKernel { input.shape(), &input_copy), done); auto input_copy_reshaped = input_copy.template flat_inner_dims(); - auto input_reshaped = input.template flat_inner_dims(); const GPUDevice& device = context->eigen_device(); if (!adjoint_) { device.memcpy(input_copy.flat().data(), input.flat().data(), input.NumElements() * sizeof(Scalar)); } else { - functor::AdjointBatchFunctor functor; - functor(device, input_reshaped, input_copy_reshaped); + OP_REQUIRES_OK_ASYNC( + context, DoConjugateMatrixTranspose(device, input, &input_copy), + done); } const int64 batch_size = input_copy_reshaped.dimension(0); @@ -238,10 +239,7 @@ class MatrixInverseOpGpu : public AsyncOpKernel { done); } } - // Callback for checking info after kernels finish. Also capture the - // temporary Tensors/ScratchSpace so they don't get deallocated before the - // kernels run. TODO(rmlarsen): Use move capture once C++14 becomes - // available. + // Callback for checking info after kernels finish. auto info_checker = [context, done]( const Status& status, const std::vector& host_infos) { diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/matrix_solve_op.cc index 862033e9fab00912bf821b359a7f6cad1859474e..169f3dae76d2fb6d0515d22648a9047657af0032 100644 --- a/tensorflow/core/kernels/matrix_solve_op.cc +++ b/tensorflow/core/kernels/matrix_solve_op.cc @@ -44,18 +44,12 @@ static const char kErrMsg[] = "Input matrix is not invertible."; template class MatrixSolveOp : public LinearAlgebraOp { public: - typedef LinearAlgebraOp Base; + INHERIT_LINALG_TYPEDEFS(Scalar); explicit MatrixSolveOp(OpKernelConstruction* context) : Base(context) { OP_REQUIRES_OK(context, context->GetAttr("adjoint", &adjoint_)); } - using TensorShapes = typename Base::TensorShapes; - using Matrix = typename Base::Matrix; - using MatrixMaps = typename Base::MatrixMaps; - using ConstMatrixMap = typename Base::ConstMatrixMap; - using ConstMatrixMaps = typename Base::ConstMatrixMaps; - void ValidateInputMatrixShapes( OpKernelContext* context, const TensorShapes& input_matrix_shapes) const final { @@ -102,7 +96,6 @@ class MatrixSolveOp : public LinearAlgebraOp { // a result of basic user mistakes such providing integer valued // matrices that are exactly singular, or due to underflow if this // code is run with denormals being flushed to zero. - using RealScalar = typename Base::RealScalar; const RealScalar min_abs_pivot = lu_decomposition.matrixLU().diagonal().cwiseAbs().minCoeff(); OP_REQUIRES(context, min_abs_pivot > RealScalar(0), @@ -181,9 +174,6 @@ class MatrixSolveOpGpu : public AsyncOpKernel { // false, try to reuse the input buffer if this op owns it exclusively. Tensor input_copy; const GPUDevice& device = context->eigen_device(); - std::vector perm(ndims); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[ndims - 2], perm[ndims - 1]); if (adjoint_) { // For the adjoint case, it is simpler to always make a transposed copy up // front. @@ -193,7 +183,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel { input.shape(), &input_copy), done); OP_REQUIRES_OK_ASYNC(context, - DoTranspose(device, input, perm, &input_copy), done); + DoMatrixTranspose(device, input, &input_copy), done); } else { OP_REQUIRES_OK_ASYNC( context, @@ -267,7 +257,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel { done); if (nrhs > 1) { OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, rhs, perm, &transposed_rhs), done); + context, DoMatrixTranspose(device, rhs, &transposed_rhs), done); } else { device.memcpy(transposed_rhs.flat().data(), rhs.flat().data(), @@ -327,7 +317,7 @@ class MatrixSolveOpGpu : public AsyncOpKernel { // 4. Transpose X to get the final result in row-major form. if (nrhs > 1) { OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, transposed_rhs, perm, output), done); + context, DoMatrixTranspose(device, transposed_rhs, output), done); } else { device.memcpy(output->flat().data(), transposed_rhs.flat().data(), diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op.cc b/tensorflow/core/kernels/matrix_triangular_solve_op.cc index 953f37fa029862fcbf1922c489af05cec8cbc6bb..6f7e6a7496840f027c08afa65d9381afe1f78a76 100644 --- a/tensorflow/core/kernels/matrix_triangular_solve_op.cc +++ b/tensorflow/core/kernels/matrix_triangular_solve_op.cc @@ -47,7 +47,7 @@ perftools::gputools::DeviceMemory AsDeviceMemory( template class MatrixTriangularSolveOp : public LinearAlgebraOp { public: - typedef LinearAlgebraOp Base; + INHERIT_LINALG_TYPEDEFS(Scalar); explicit MatrixTriangularSolveOp(OpKernelConstruction* context) : Base(context), lower_(true), adjoint_(false) { @@ -55,13 +55,6 @@ class MatrixTriangularSolveOp : public LinearAlgebraOp { OP_REQUIRES_OK(context, context->GetAttr("adjoint", &adjoint_)); } - using TensorShapes = typename Base::TensorShapes; - using Matrix = typename Base::Matrix; - using MatrixMap = typename Base::MatrixMap; - using MatrixMaps = typename Base::MatrixMaps; - using ConstMatrixMap = typename Base::ConstMatrixMap; - using ConstMatrixMaps = typename Base::ConstMatrixMaps; - void ValidateInputMatrixShapes( OpKernelContext* context, const TensorShapes& input_matrix_shapes) const final { @@ -97,7 +90,6 @@ class MatrixTriangularSolveOp : public LinearAlgebraOp { // an empty set of equation as the empty matrix. return; } - using RealScalar = typename Base::RealScalar; const RealScalar min_abs_pivot = matrix.diagonal().cwiseAbs().minCoeff(); OP_REQUIRES(context, min_abs_pivot > RealScalar(0), errors::InvalidArgument("Input matrix is not invertible.")); diff --git a/tensorflow/core/kernels/mirror_pad_op.cc b/tensorflow/core/kernels/mirror_pad_op.cc index e3643f9447bf611a6049b4cc048e3d758eaf983c..fbdeaf43ebbfdcf6b76f97046130f40cf8c8efd1 100644 --- a/tensorflow/core/kernels/mirror_pad_op.cc +++ b/tensorflow/core/kernels/mirror_pad_op.cc @@ -18,10 +18,10 @@ limitations under the License. #define EIGEN_USE_THREADS #include "tensorflow/core/kernels/mirror_pad_op.h" - #include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -35,7 +35,7 @@ limitations under the License. namespace tensorflow { -template +template class MirrorPadOp : public OpKernel { public: explicit MirrorPadOp(OpKernelConstruction* context) : OpKernel(context) { @@ -82,10 +82,10 @@ class MirrorPadOp : public OpKernel { // Compute the shape of the output tensor, and allocate it. TensorShape output_shape; - TTypes::ConstMatrix paddings = in1.matrix(); + typename TTypes::ConstMatrix paddings = in1.matrix(); for (int d = 0; d < dims; ++d) { - const int32 before = paddings(d, 0); // Pad before existing elements. - const int32 after = paddings(d, 1); // Pad after existing elements. + const Tpaddings before = paddings(d, 0); // Pad before existing elements. + const Tpaddings after = paddings(d, 1); // Pad after existing elements. OP_REQUIRES(context, before >= 0 && after >= 0, errors::InvalidArgument("paddings must be non-negative: ", before, " ", after)); @@ -121,7 +121,7 @@ class MirrorPadOp : public OpKernel { #define MIRROR_PAD_CASE(i) \ case i: { \ - functor::MirrorPad()( \ + functor::MirrorPad()( \ context->eigen_device(), To32Bit(output->tensor()), \ To32Bit(in0.tensor()), paddings, offset_); \ break; \ @@ -152,20 +152,25 @@ using GpuDevice = Eigen::GpuDevice; namespace functor { // Forward declarations of the functor specializations defined in the sharded // files. -#define DECLARE_CPU_SPEC(T, i) \ - template <> \ - void MirrorPad::operator()( \ - const CpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int); \ - extern template struct MirrorPad; - -#define DECLARE_CPU_SPECS(T) \ - DECLARE_CPU_SPEC(T, 1); \ - DECLARE_CPU_SPEC(T, 2); \ - DECLARE_CPU_SPEC(T, 3); \ - DECLARE_CPU_SPEC(T, 4); \ - DECLARE_CPU_SPEC(T, 5); +#define DECLARE_CPU_SPEC(T, Tpaddings, i) \ + template <> \ + void MirrorPad::operator()( \ + const CpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int); \ + extern template struct MirrorPad; + +#define DECLARE_CPU_SPECS(T) \ + DECLARE_CPU_SPEC(T, int32, 1); \ + DECLARE_CPU_SPEC(T, int32, 2); \ + DECLARE_CPU_SPEC(T, int32, 3); \ + DECLARE_CPU_SPEC(T, int32, 4); \ + DECLARE_CPU_SPEC(T, int32, 5); \ + DECLARE_CPU_SPEC(T, int64, 1); \ + DECLARE_CPU_SPEC(T, int64, 2); \ + DECLARE_CPU_SPEC(T, int64, 3); \ + DECLARE_CPU_SPEC(T, int64, 4); \ + DECLARE_CPU_SPEC(T, int64, 5); TF_CALL_POD_TYPES(DECLARE_CPU_SPECS); @@ -179,7 +184,13 @@ TF_CALL_POD_TYPES(DECLARE_CPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - MirrorPadOp); + MirrorPadOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadOp); // Note that we do register for bool type, but not in the gradient op. TF_CALL_POD_TYPES(REGISTER_KERNEL); @@ -188,20 +199,25 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL); #if GOOGLE_CUDA namespace functor { // Forward declarations of the functor specializations for GPU. -#define DECLARE_GPU_SPEC(T, i) \ - template <> \ - void MirrorPad::operator()( \ - const GpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int); \ - extern template struct MirrorPad; - -#define DECLARE_GPU_SPECS(T) \ - DECLARE_GPU_SPEC(T, 1); \ - DECLARE_GPU_SPEC(T, 2); \ - DECLARE_GPU_SPEC(T, 3); \ - DECLARE_GPU_SPEC(T, 4); \ - DECLARE_GPU_SPEC(T, 5); +#define DECLARE_GPU_SPEC(T, Tpaddings, i) \ + template <> \ + void MirrorPad::operator()( \ + const GpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int); \ + extern template struct MirrorPad; + +#define DECLARE_GPU_SPECS(T) \ + DECLARE_GPU_SPEC(T, int32, 1); \ + DECLARE_GPU_SPEC(T, int32, 2); \ + DECLARE_GPU_SPEC(T, int32, 3); \ + DECLARE_GPU_SPEC(T, int32, 4); \ + DECLARE_GPU_SPEC(T, int32, 5); \ + DECLARE_GPU_SPEC(T, int64, 1); \ + DECLARE_GPU_SPEC(T, int64, 2); \ + DECLARE_GPU_SPEC(T, int64, 3); \ + DECLARE_GPU_SPEC(T, int64, 4); \ + DECLARE_GPU_SPEC(T, int64, 5); TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); #undef DECLARE_GPU_SPECS @@ -215,14 +231,20 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - MirrorPadOp) + MirrorPadOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); #undef REGISTER_GPU_KERNEL #endif // GOOGLE_CUDA // Gradient op. -template +template class MirrorPadGradOp : public OpKernel { public: explicit MirrorPadGradOp(OpKernelConstruction* context) : OpKernel(context) { @@ -269,10 +291,10 @@ class MirrorPadGradOp : public OpKernel { // Compute the shape of the output tensor, and allocate it. TensorShape output_shape; - TTypes::ConstMatrix paddings = in1.matrix(); + typename TTypes::ConstMatrix paddings = in1.matrix(); for (int d = 0; d < dims; ++d) { - const int32 before = paddings(d, 0); // Pad before existing elements. - const int32 after = paddings(d, 1); // Pad after existing elements. + const Tpaddings before = paddings(d, 0); // Pad before existing elements. + const Tpaddings after = paddings(d, 1); // Pad after existing elements. OP_REQUIRES(context, before >= 0 && after >= 0, errors::InvalidArgument("Paddings must be non-negative: ", before, ", ", after)); @@ -308,7 +330,7 @@ class MirrorPadGradOp : public OpKernel { #define MIRROR_PAD_GRAD_CASE(k) \ case k: { \ - functor::MirrorPadGrad()( \ + functor::MirrorPadGrad()( \ context->eigen_device(), To32Bit(output->tensor()), \ To32Bit(in0.tensor()), paddings, offset_, \ To32Bit(scratch.tensor())); \ @@ -337,33 +359,45 @@ class MirrorPadGradOp : public OpKernel { namespace functor { // Forward declarations of the functor specializations defined in the sharded // files. -#define DECLARE_CPU_SPEC(T, k) \ - template <> \ - void MirrorPadGrad::operator()( \ - const CpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int, typename TTypes::Tensor); \ - extern template struct MirrorPadGrad; - -#define DECLARE_CPU_SPECS(T) \ - DECLARE_CPU_SPEC(T, 1); \ - DECLARE_CPU_SPEC(T, 2); \ - DECLARE_CPU_SPEC(T, 3); \ - DECLARE_CPU_SPEC(T, 4); \ - DECLARE_CPU_SPEC(T, 5); +#define DECLARE_CPU_SPEC(T, Tpaddings, k) \ + template <> \ + void MirrorPadGrad::operator()( \ + const CpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int, \ + typename TTypes::Tensor); \ + extern template struct MirrorPadGrad; + +#define DECLARE_CPU_SPECS(T) \ + DECLARE_CPU_SPEC(T, int32, 1); \ + DECLARE_CPU_SPEC(T, int32, 2); \ + DECLARE_CPU_SPEC(T, int32, 3); \ + DECLARE_CPU_SPEC(T, int32, 4); \ + DECLARE_CPU_SPEC(T, int32, 5); \ + DECLARE_CPU_SPEC(T, int64, 1); \ + DECLARE_CPU_SPEC(T, int64, 2); \ + DECLARE_CPU_SPEC(T, int64, 3); \ + DECLARE_CPU_SPEC(T, int64, 4); \ + DECLARE_CPU_SPEC(T, int64, 5); TF_CALL_NUMBER_TYPES(DECLARE_CPU_SPECS); #undef DECLARE_CPU_SPECS #undef DECLARE_CPU_SPEC } // namespace functor -#define REGISTER_KERNEL(type) \ - REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("paddings"), \ - MirrorPadGradOp); +#define REGISTER_KERNEL(type) \ + REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadGradOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadGradOp); TF_CALL_NUMBER_TYPES(REGISTER_KERNEL); #undef REGISTER_KERNEL @@ -371,20 +405,26 @@ TF_CALL_NUMBER_TYPES(REGISTER_KERNEL); #if GOOGLE_CUDA namespace functor { // Forward declarations of the functor specializations for GPU. -#define DECLARE_GPU_SPEC(T, k) \ - template <> \ - void MirrorPadGrad::operator()( \ - const GpuDevice&, typename TTypes::Tensor, \ - typename TTypes::ConstTensor, TTypes::ConstMatrix, \ - int, typename TTypes::Tensor); \ - extern template struct MirrorPadGrad; - -#define DECLARE_GPU_SPECS(T) \ - DECLARE_GPU_SPEC(T, 1); \ - DECLARE_GPU_SPEC(T, 2); \ - DECLARE_GPU_SPEC(T, 3); \ - DECLARE_GPU_SPEC(T, 4); \ - DECLARE_GPU_SPEC(T, 5); +#define DECLARE_GPU_SPEC(T, Tpaddings, k) \ + template <> \ + void MirrorPadGrad::operator()( \ + const GpuDevice&, typename TTypes::Tensor, \ + typename TTypes::ConstTensor, \ + TTypes::ConstMatrix, int, \ + typename TTypes::Tensor); \ + extern template struct MirrorPadGrad; + +#define DECLARE_GPU_SPECS(T) \ + DECLARE_GPU_SPEC(T, int32, 1); \ + DECLARE_GPU_SPEC(T, int32, 2); \ + DECLARE_GPU_SPEC(T, int32, 3); \ + DECLARE_GPU_SPEC(T, int32, 4); \ + DECLARE_GPU_SPEC(T, int32, 5); \ + DECLARE_GPU_SPEC(T, int64, 1); \ + DECLARE_GPU_SPEC(T, int64, 2); \ + DECLARE_GPU_SPEC(T, int64, 3); \ + DECLARE_GPU_SPEC(T, int64, 4); \ + DECLARE_GPU_SPEC(T, int64, 5); TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); #undef DECLARE_GPU_SPECS @@ -398,7 +438,13 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - MirrorPadGradOp) + MirrorPadGradOp); \ + REGISTER_KERNEL_BUILDER(Name("MirrorPadGrad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + MirrorPadGradOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); #undef REGISTER_GPU_KERNEL diff --git a/tensorflow/core/kernels/mirror_pad_op.h b/tensorflow/core/kernels/mirror_pad_op.h index b83d2223d0b2fd7e8500b2b10c47980aab9c407e..81150a9e791fee5eb0bac80d4221bd3dd572ddbb 100644 --- a/tensorflow/core/kernels/mirror_pad_op.h +++ b/tensorflow/core/kernels/mirror_pad_op.h @@ -64,9 +64,8 @@ class TensorMirrorPadOp StorageKind; typedef typename Eigen::internal::traits::Index Index; - EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE - TensorMirrorPadOp(const XprType& expr, const PaddingDimensions& padding_dims, - Index offset) + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TensorMirrorPadOp( + const XprType& expr, const PaddingDimensions& padding_dims, Index offset) : xpr_(expr), padding_dims_(padding_dims), offset_(offset) {} EIGEN_DEVICE_FUNC @@ -336,12 +335,12 @@ namespace functor { // offset argument must be either 0 or 1. This controls whether the boundary // values are replicated (offset == 0) or not replicated (offset == 1). -template +template struct MirrorPad { void operator()(const Device& device, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - TTypes::ConstMatrix padding, int offset) { + typename TTypes::ConstMatrix padding, int offset) { Eigen::array, Dims> padding_dims; for (int i = 0; i < Dims; ++i) { @@ -363,12 +362,12 @@ struct MirrorPad { // offset argument must be either 0 or 1. This controls whether the boundary // values are replicated (offset == 0) or not replicated (offset == 1). -template +template struct MirrorPadGrad { void operator()(const Device& device, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - TTypes::ConstMatrix paddings, int offset, + typename TTypes::ConstMatrix paddings, int offset, typename TTypes::Tensor scratch) { // Copy the gradient input into the scratch buffer. scratch.device(device) = input; diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h index 9864f5633a34be1bc78bb6367a5f38244b9e7b4b..bb22b2aa918dad379b80931ba0893feb9366489b 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h +++ b/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h @@ -25,13 +25,17 @@ namespace tensorflow { using CpuDevice = Eigen::ThreadPoolDevice; -#define DEFINE_CPU_SPECS(T) \ - template struct functor::MirrorPad; +#define DEFINE_CPU_SPECS(T) \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; TF_CALL_POD_TYPES(DEFINE_CPU_SPECS); #undef DEFINE_CPU_SPECS -#define DEFINE_CPU_SPECS(T) \ - template struct functor::MirrorPadGrad; +#define DEFINE_CPU_SPECS(T) \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; TF_CALL_NUMBER_TYPES(DEFINE_CPU_SPECS); #undef DEFINE_CPU_SPECS diff --git a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc index 8074aa96243a4f24b3c9a9e976405b246e8e0056..dbd0a9bd8f94293092443e7fd7c14b52e758dd37 100644 --- a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc +++ b/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc @@ -25,17 +25,27 @@ namespace tensorflow { using GpuDevice = Eigen::GpuDevice; -#define DEFINE_GPU_SPECS(T) \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; \ - template struct functor::MirrorPadGrad; +#define DEFINE_GPU_SPECS(T) \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; \ + template struct functor::MirrorPadGrad; TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS); #undef DEFINE_GPU_SPECS diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc index 57661e8b105a3ad10d3fd8d9f70da2f9275364bb..369f632fb4623347e5a808443397ecec87b6cfa8 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl_conv_ops.cc @@ -288,8 +288,10 @@ class MklConv2DOp : public OpKernel { mkl_filter_output_mkl_shape.SetMklLayout(mkl_context.prim_fwd, dnnResourceFilter); - size_t filter_sizes[4] = {filter.dim_size(0), filter.dim_size(1), - filter.dim_size(2), filter.dim_size(3)}; + size_t filter_sizes[4] = {static_cast(filter.dim_size(0)), + static_cast(filter.dim_size(1)), + static_cast(filter.dim_size(2)), + static_cast(filter.dim_size(3))}; mkl_filter_output_mkl_shape.SetTfLayout(filter.dims(), filter_sizes, mkl_context.filter_strides); diff --git a/tensorflow/core/kernels/mkl_transpose_op.cc b/tensorflow/core/kernels/mkl_transpose_op.cc index 89a1d5e8a7da50876df74a1b98e8485eadf50655..764d4c9400e5751de29b9651eebc1328fdd09d59 100644 --- a/tensorflow/core/kernels/mkl_transpose_op.cc +++ b/tensorflow/core/kernels/mkl_transpose_op.cc @@ -18,6 +18,9 @@ limitations under the License. #ifdef INTEL_MKL #define EIGEN_USE_THREADS +#include "tensorflow/core/framework/numeric_types.h" +#define MKL_Complex8 tensorflow::complex64 +#define MKL_Complex16 tensorflow::complex128 #include "mkl_trans.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/kernels/transpose_op.h" @@ -41,7 +44,7 @@ namespace tensorflow { namespace { template -void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {} +Status MKLTranspose2D(const char trans, const Tensor& in, Tensor* out); // Documentation here: https://software.intel.com/en-us/node/520863 // Parameters: (ordering:row-major, operation:transpose, num_rows, num_cols, @@ -54,70 +57,73 @@ void MKLTranspose2D(const char trans, const Tensor& in, Tensor* out) {} mkl_##PREFIX##omatcopy('R', trans, in.dim_size(0), in.dim_size(1), 1, \ in.flat().data(), in.dim_size(1), \ out->flat().data(), in.dim_size(0)); \ - return Status::OK(); + return Status::OK(); \ } - INSTANTIATE(float, s) - INSTANTIATE(double, d) - INSTANTIATE(complex64, c) - INSTANTIATE(complex128, z) +INSTANTIATE(float, s) +INSTANTIATE(double, d) +INSTANTIATE(complex64, c) +INSTANTIATE(complex128, z) #undef INSTANTIATE - static const char kMKLTranspose = 'T'; - static const char kMKLConjugateTranspose = 'C'; - - } // namespace tensorflow - - Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in, - gtl::ArraySlice perm, - Tensor* out) { - if (in.dims() == 2) { - switch (in.dtype()) { - case DT_FLOAT: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_DOUBLE: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_COMPLEX64: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_COMPLEX128: - return MKLTranspose2D(kMKLTranspose, in, out); - default: - break; - } +static const char kMKLTranspose = 'T'; +static const char kMKLConjugateTranspose = 'C'; + +} // namespace + +Status MklTransposeCpuOp::DoTranspose(OpKernelContext* ctx, const Tensor& in, + gtl::ArraySlice perm, + Tensor* out) { + if (in.dims() == 2) { + if (perm[0] == 0 && perm[1] == 1) { + return Status::OK(); + } + switch (in.dtype()) { + case DT_FLOAT: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_DOUBLE: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_COMPLEX64: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_COMPLEX128: + return MKLTranspose2D(kMKLTranspose, in, out); + default: + break; } - // Fallback to eigen if transpose parameters not supported by MKL - typedef Eigen::ThreadPoolDevice CPUDevice; - return ::tensorflow::DoTranspose(ctx->eigen_device(), in, perm, - out); } - - Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx, - const Tensor& in, - gtl::ArraySlice perm, - Tensor* out) { - if (in.dims() == 2) { - // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels - // for any transpose that can be reduced to swapping the last two - // dimensions in a rank-3 tensor. We can even run each outer dimension in - // a separate thread. - switch (in.dtype()) { - case DT_FLOAT: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_DOUBLE: - return MKLTranspose2D(kMKLTranspose, in, out); - case DT_COMPLEX64: - return MKLTranspose2D(kMKLConjugateTranspose, in, out); - case DT_COMPLEX128: - return MKLTranspose2D(kMKLConjugateTranspose, in, out); - default: - break; - } + // Fallback to eigen if transpose parameters not supported by MKL + typedef Eigen::ThreadPoolDevice CPUDevice; + return ::tensorflow::DoTranspose(ctx->eigen_device(), in, perm, + out); +} + +Status MklConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx, + const Tensor& in, + gtl::ArraySlice perm, + Tensor* out) { + if (in.dims() == 2 && perm[0] == 1 && perm[1] == 0) { + // TODO(rmlarsen): By setting lda and ldb, we could use the MKL kernels + // for any transpose that can be reduced to swapping the last two + // dimensions in a rank-3 tensor. We can even run each outer dimension in + // a separate thread. + switch (in.dtype()) { + case DT_FLOAT: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_DOUBLE: + return MKLTranspose2D(kMKLTranspose, in, out); + case DT_COMPLEX64: + return MKLTranspose2D(kMKLConjugateTranspose, in, out); + case DT_COMPLEX128: + return MKLTranspose2D(kMKLConjugateTranspose, in, out); + default: + break; } - // Fallback to eigen if transpose parameters not supported by MKL - typedef Eigen::ThreadPoolDevice CPUDevice; - return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), - in, perm, out); } + // Fallback to eigen if transpose parameters not supported by MKL + typedef Eigen::ThreadPoolDevice CPUDevice; + return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), in, + perm, out); +} } // namespace tensorflow diff --git a/tensorflow/core/kernels/nth_element_op.cc b/tensorflow/core/kernels/nth_element_op.cc new file mode 100644 index 0000000000000000000000000000000000000000..da825e408c24617862e8613c6b63ed1a51944041 --- /dev/null +++ b/tensorflow/core/kernels/nth_element_op.cc @@ -0,0 +1,139 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// See docs in ../ops/nn_ops.cc. +#include "tensorflow/core/kernels/nth_element_op.h" + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/work_sharder.h" +#include +#include +#include + +namespace tensorflow { + +typedef Eigen::ThreadPoolDevice CPUDevice; + +template +class NthElementOp : public OpKernel { + public: + explicit NthElementOp(OpKernelConstruction* context) : OpKernel(context) { + OP_REQUIRES_OK(context, context->GetAttr("reverse", &reverse_)); + } + + void Compute(OpKernelContext* context) override { + // The second args is N, which must be a positive scalar. + const auto& n_in = context->input(1); + OP_REQUIRES(context, TensorShapeUtils::IsScalar(n_in.shape()), + errors::InvalidArgument("N must be scalar, got shape ", + n_in.shape().DebugString())); + int n = n_in.scalar()(); + OP_REQUIRES(context, n >= 0, + errors::InvalidArgument("Need n >= 0, got ", n)); + + // The first args is input tensor, which must have 1 dimension at least. + const Tensor& input_in = context->input(0); + const int num_dims = input_in.dims(); + OP_REQUIRES(context, num_dims >= 1, + errors::InvalidArgument("Input must be >= 1-D, got shape ", + input_in.shape().DebugString())); + // The last dimension of input tensor must be greater than N. + OP_REQUIRES(context, input_in.dim_size(num_dims-1) > n, + errors::InvalidArgument("Input must have at least n+1 columns")); + + // std::nth_element only support the nth-smallest selection. + if (reverse_) { + n = input_in.dim_size(num_dims - 1) - n - 1; + } + + // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1]. + TensorShape out_shape; + for (int i = 0; i < num_dims-1; ++i) { + out_shape.AddDim(input_in.dim_size(i)); + } + Tensor* output_tensor = nullptr; + OP_REQUIRES_OK(context, + context->allocate_output(0, out_shape, &output_tensor)); + + functor::NthElementFunctor nthElementFunc; + nthElementFunc(context, input_in, *output_tensor, n, reverse_); + } + + private: + bool reverse_; +}; + +namespace functor { + +template +struct NthElementFunctor { + void operator() (OpKernelContext* context, + const Tensor& input_tensor, + Tensor& output_tensor, + int n, + bool reverse) { + const T* input = input_tensor.flat().data(); + T* output = output_tensor.flat().data(); + + // Assume input_shape is [d1,d2,...dk], and output_shape is [d1,d2...dk-1], + // then num_rows = d1*d2...dk-1, last_dim = dk. + const int num_rows = output_tensor.NumElements(); + const int last_dim = input_tensor.dim_size(input_tensor.dims()-1); + + // Allocate each row to different shard. + auto SubNthElement = [&, input, output, last_dim, n](int start, + int limit) { + // std::nth_element would rearrange the array, so we need a new buffer. + std::vector buf(last_dim); + + for (int b = start; b < limit; ++b) { + // Copy from one row of elements to buffer + const T* input_start = input + b * last_dim; + const T* input_end = input + (b+1) * last_dim; + std::copy(input_start, input_end, buf.begin()); + + std::nth_element(buf.begin(), buf.begin()+n, buf.end()); + // The element placed in the nth position is exactly the element that + // would occur in this position if the range was fully sorted. + output[b] = buf[n]; + } + }; + + auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads()); + // The average time complexity of partition-based nth_element (BFPRT) is O(n), + // althought the worst time complexity could be O(n^2). + // Here, 20 is a empirical factor of cost_per_unit. + Shard(worker_threads.num_threads, worker_threads.workers, num_rows, + 20 * last_dim, SubNthElement); + } +}; + +} // namespace functor + + +#define REGISTER_NTHOP(T) \ + REGISTER_KERNEL_BUILDER( \ + Name("NthElement").Device(DEVICE_CPU).TypeConstraint("T"), \ + NthElementOp) + +TF_CALL_REAL_NUMBER_TYPES(REGISTER_NTHOP); +#undef REGISTER_NTHOP + +} // end namespace tensorflow + diff --git a/tensorflow/core/kernels/nth_element_op.h b/tensorflow/core/kernels/nth_element_op.h new file mode 100644 index 0000000000000000000000000000000000000000..11a6c996b093fa7255a230122f64eb1054789453 --- /dev/null +++ b/tensorflow/core/kernels/nth_element_op.h @@ -0,0 +1,39 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_NTH_ELEMENT_OP_H_ +#define TENSORFLOW_NTH_ELEMENT_OP_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +namespace functor { + +template +struct NthElementFunctor { + void operator() (OpKernelContext* context, + const Tensor& input_tensor, + Tensor& output_tensor, + int n); +}; + +} // namespace functor + +} // namespace tensorflow + +#endif // TENSORFLOW_NTH_ELEMENT_OP_H_ diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc index 6196c5ed93ee3c0ff4001ad2b1d3bb7ac2776022..eff3e4d92cc3ecc3b172a970564225c8b204cdd4 100644 --- a/tensorflow/core/kernels/pad_op.cc +++ b/tensorflow/core/kernels/pad_op.cc @@ -40,9 +40,9 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL -template +template class PadOp : public OpKernel { public: explicit PadOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -82,10 +82,11 @@ class PadOp : public OpKernel { // Compute the shape of the output tensor, and allocate it. TensorShape output_shape; - TTypes::ConstMatrix paddings = in1.matrix(); + typename TTypes::ConstMatrix paddings = in1.matrix(); for (int d = 0; d < fixed_dims; ++d) { - const int32 before_d = paddings(d, 0); // Pad before existing elements. - const int32 after_d = paddings(d, 1); // Pad after existing elements. + const Tpadding before_d = + paddings(d, 0); // Pad before existing elements. + const Tpadding after_d = paddings(d, 1); // Pad after existing elements. OP_REQUIRES(context, before_d >= 0 && after_d >= 0, errors::InvalidArgument("Paddings must be non-negative: ", before_d, " ", after_d)); @@ -142,32 +143,47 @@ class PadOp : public OpKernel { template void Operate(OpKernelContext* context, typename TTypes::ConstTensor input, - TTypes::ConstMatrix paddings, T pad_value, + typename TTypes::ConstMatrix paddings, T pad_value, Tensor* output) { CHECK_EQ(Dims, paddings.dimension(0)); CHECK_EQ(2, paddings.dimension(1)); - Eigen::array, Dims> paddings_array; + Eigen::array, Dims> paddings_array; for (int i = 0; i < Dims; ++i) { paddings_array[i] = {paddings(i, 0), paddings(i, 1)}; } - functor::Pad functor; + functor::Pad functor; functor(context->eigen_device(), output->tensor(), input, paddings_array, pad_value); } }; -#define REGISTER_KERNEL(type) \ - REGISTER_KERNEL_BUILDER(Name("Pad") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .HostMemory("paddings"), \ - PadOp); \ - REGISTER_KERNEL_BUILDER(Name("PadV2") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .HostMemory("paddings") \ - .HostMemory("constant_values"), \ - PadOp); +#define REGISTER_KERNEL(type) \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp); TF_CALL_POD_TYPES(REGISTER_KERNEL); #undef REGISTER_KERNEL @@ -177,11 +193,17 @@ TF_CALL_POD_TYPES(REGISTER_KERNEL); namespace functor { #define DECLARE_GPU_SPEC(T, Dims) \ template <> \ - void Pad::operator()( \ + void Pad::operator()( \ const GPUDevice& d, typename TTypes::Tensor output, \ typename TTypes::ConstTensor input, \ Eigen::array, Dims> paddings, T pad_value); \ - extern template struct Pad; + extern template struct Pad; \ + template <> \ + void Pad::operator()( \ + const GPUDevice& d, typename TTypes::Tensor output, \ + typename TTypes::ConstTensor input, \ + Eigen::array, Dims> paddings, T pad_value); \ + extern template struct Pad; #define DECLARE_GPU_SPECS(T) \ DECLARE_GPU_SPEC(T, 0); \ @@ -202,14 +224,27 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_GPU_SPECS); .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - PadOp); \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ REGISTER_KERNEL_BUILDER(Name("PadV2") \ .Device(DEVICE_GPU) \ .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings") \ .HostMemory("constant_values"), \ - PadOp) + PadOp) \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNEL); @@ -223,7 +258,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad") .HostMemory("input") .HostMemory("paddings") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("Pad") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("output"), + PadOp); REGISTER_KERNEL_BUILDER(Name("PadV2") .Device(DEVICE_GPU) .TypeConstraint("T") @@ -232,7 +275,16 @@ REGISTER_KERNEL_BUILDER(Name("PadV2") .HostMemory("paddings") .HostMemory("constant_values") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("PadV2") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("constant_values") + .HostMemory("output"), + PadOp); #endif #ifdef TENSORFLOW_USE_SYCL @@ -243,14 +295,27 @@ REGISTER_KERNEL_BUILDER(Name("PadV2") .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings"), \ - PadOp); \ + PadOp); \ + REGISTER_KERNEL_BUILDER(Name("Pad") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings"), \ + PadOp); \ REGISTER_KERNEL_BUILDER(Name("PadV2") \ .Device(DEVICE_SYCL) \ .TypeConstraint("T") \ .TypeConstraint("Tpaddings") \ .HostMemory("paddings") \ .HostMemory("constant_values"), \ - PadOp) + PadOp) \ + REGISTER_KERNEL_BUILDER(Name("PadV2") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tpaddings") \ + .HostMemory("paddings") \ + .HostMemory("constant_values"), \ + PadOp) TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL_KERNEL); REGISTER_KERNEL_BUILDER(Name("Pad") @@ -260,7 +325,15 @@ REGISTER_KERNEL_BUILDER(Name("Pad") .HostMemory("input") .HostMemory("paddings") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("Pad") + .Device(DEVICE_SYCL) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("output"), + PadOp); REGISTER_KERNEL_BUILDER(Name("PadV2") .Device(DEVICE_SYCL) .TypeConstraint("T") @@ -269,8 +342,17 @@ REGISTER_KERNEL_BUILDER(Name("PadV2") .HostMemory("paddings") .HostMemory("constant_values") .HostMemory("output"), - PadOp); + PadOp); +REGISTER_KERNEL_BUILDER(Name("PadV2") + .Device(DEVICE_SYCL) + .TypeConstraint("T") + .TypeConstraint("Tpaddings") + .HostMemory("input") + .HostMemory("paddings") + .HostMemory("constant_values") + .HostMemory("output"), + PadOp); #undef REGISTER_SYCL_KERNEL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // end namespace tensorflow diff --git a/tensorflow/core/kernels/pad_op.h b/tensorflow/core/kernels/pad_op.h index 95a7c9a3ae58b66fd7711a31aa90365aef5a4a46..ee9e0f033058c0ba783d40d588f654573e287db4 100644 --- a/tensorflow/core/kernels/pad_op.h +++ b/tensorflow/core/kernels/pad_op.h @@ -25,13 +25,13 @@ namespace tensorflow { namespace functor { // Functor used by PadOp to do the computations. -template +template struct Pad { // Pad "input" into "output", as specified by "paddings" and "pad_value". // See pad_op.cc for details. void operator()(const Device& d, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - Eigen::array, Dims> paddings, + Eigen::array, Dims> paddings, T pad_value) { if (Eigen::internal::is_same::value && (output.size() <= std::numeric_limits::max())) { @@ -42,12 +42,12 @@ struct Pad { } }; -template -struct Pad { +template +struct Pad { // In the scalar case we simply copy the input. void operator()(const Device& d, typename TTypes::Tensor output, typename TTypes::ConstTensor input, - Eigen::array, 0>, T) { + Eigen::array, 0>, T) { output.device(d) = input; } }; diff --git a/tensorflow/core/kernels/pad_op_gpu.cu.cc b/tensorflow/core/kernels/pad_op_gpu.cu.cc index f98631df17b1445528eeb56a3e9f9b4a99c011e9..613ad628251915951be7a99ce687ceeef89d7aef 100644 --- a/tensorflow/core/kernels/pad_op_gpu.cu.cc +++ b/tensorflow/core/kernels/pad_op_gpu.cu.cc @@ -26,14 +26,18 @@ namespace tensorflow { typedef Eigen::GpuDevice GPUDevice; // Definition of the GPU implementations declared in pad_op.cc. -#define DEFINE_GPU_SPECS(T) \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; \ - template struct functor::Pad; +#define DEFINE_GPU_PAD_SPECS(T, Tpadding) \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; \ + template struct functor::Pad; + +#define DEFINE_GPU_SPECS(T) \ + DEFINE_GPU_PAD_SPECS(T, int32) \ + DEFINE_GPU_PAD_SPECS(T, int64) TF_CALL_GPU_NUMBER_TYPES(DEFINE_GPU_SPECS); diff --git a/tensorflow/core/kernels/padded_batch_dataset_op.cc b/tensorflow/core/kernels/padded_batch_dataset_op.cc index 7737f57b6875275d8e752c0f77d64450c4dfa70c..cfc77690b568a3223ca33f359f47fe22de9b35ff 100644 --- a/tensorflow/core/kernels/padded_batch_dataset_op.cc +++ b/tensorflow/core/kernels/padded_batch_dataset_op.cc @@ -349,7 +349,6 @@ class PaddedBatchDatasetOp : public UnaryDatasetOpKernel { private: mutex mu_; - int64 i_ GUARDED_BY(mu_); std::unique_ptr input_impl_ GUARDED_BY(mu_); }; diff --git a/tensorflow/core/kernels/sloppy_interleave_dataset_op.cc b/tensorflow/core/kernels/parallel_interleave_dataset_op.cc similarity index 84% rename from tensorflow/core/kernels/sloppy_interleave_dataset_op.cc rename to tensorflow/core/kernels/parallel_interleave_dataset_op.cc index 8f9f48700c17cd0a3ac11f84449ff13cdf8be66c..56942a5c01f3c2be5617aa1a9e1eadea12857911 100644 --- a/tensorflow/core/kernels/sloppy_interleave_dataset_op.cc +++ b/tensorflow/core/kernels/parallel_interleave_dataset_op.cc @@ -17,12 +17,11 @@ limitations under the License. #include "tensorflow/core/common_runtime/function.h" #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/kernels/captured_function.h" #include "tensorflow/core/kernels/dataset_utils.h" #include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/lib/random/random.h" -#include "tensorflow/core/kernels/captured_function.h" - namespace tensorflow { namespace { @@ -30,9 +29,9 @@ namespace { // See documentation in ../ops/dataset_ops.cc for a high-level // description of the following op. -class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { +class ParallelInterleaveDatasetOp : public UnaryDatasetOpKernel { public: - explicit SloppyInterleaveDatasetOp(OpKernelConstruction* ctx) + explicit ParallelInterleaveDatasetOp(OpKernelConstruction* ctx) : UnaryDatasetOpKernel(ctx), graph_def_version_(ctx->graph_def_version()) { OP_REQUIRES_OK(ctx, ctx->GetAttr("f", &func_)); @@ -62,13 +61,16 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { OP_REQUIRES(ctx, block_length > 0, errors::InvalidArgument("`block_length` must be > 0")); + bool sloppy; + OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, "sloppy", &sloppy)); + std::unique_ptr captured_func; OP_REQUIRES_OK(ctx, CapturedFunction::Create(ctx, func_, graph_def_version_, std::move(other_arguments), &captured_func)); *output = new Dataset(input, std::move(captured_func), cycle_length, - block_length, output_types_, output_shapes_); + block_length, sloppy, output_types_, output_shapes_); } private: @@ -76,12 +78,13 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { public: Dataset(const DatasetBase* input, std::unique_ptr captured_func, int64 cycle_length, - int64 block_length, const DataTypeVector& output_types, + int64 block_length, bool sloppy, const DataTypeVector& output_types, const std::vector& output_shapes) : input_(input), captured_func_(std::move(captured_func)), cycle_length_(cycle_length), block_length_(block_length), + sloppy_(sloppy), output_types_(output_types), output_shapes_(output_shapes) { input_->Ref(); @@ -91,8 +94,8 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { std::unique_ptr MakeIterator( const string& prefix) const override { - return std::unique_ptr( - new Iterator({this, strings::StrCat(prefix, "::SloppyInterleave")})); + return std::unique_ptr(new Iterator( + {this, strings::StrCat(prefix, "::ParallelInterleave")})); } const DataTypeVector& output_dtypes() const override { @@ -103,7 +106,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { } string DebugString() override { - return "SloppyInterleaveDatasetOp::Dataset"; + return "ParallelInterleaveDatasetOp::Dataset"; } private: @@ -131,16 +134,24 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { bool* end_of_sequence) override { mutex_lock l(mu_); TF_RETURN_IF_ERROR(EnsureWorkerThreadsStarted(ctx)); - // Search for available items, blocking if necessary. + const int64 num_workers = worker_threads_.size(); + if (num_workers == 0) { + *end_of_sequence = true; + return Status::OK(); + } while (!cancelled_) { - for (size_t i = 0; i < dataset()->cycle_length_; ++i) { - size_t index = (next_index_ + i) % dataset()->cycle_length_; + // Wait for an item to become available, blocking if necessary. If we + // are allowed to be sloppy, we can skip over input datasets that do + // not have an item readily available. + const int64 n = dataset()->sloppy_ ? num_workers : 1LL; + for (int64 i = 0; i < n; ++i) { + int64 index = (next_index_ + i) % num_workers; if (output_elements_[index].is_produced) { next_index_ = index; if (i == 0) { block_count_++; if (block_count_ == dataset()->block_length_) { - next_index_ = (index + 1) % dataset()->cycle_length_; + next_index_ = (index + 1) % num_workers; block_count_ = 0; } } else { @@ -150,7 +161,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { if (output_elements_[index].end_of_sequence) { output_elements_[index].is_produced = false; output_elements_[index].cond_var.notify_one(); - next_index_ = (index + 1) % dataset()->cycle_length_; + next_index_ = (index + 1) % num_workers; block_count_ = 0; i = -1; // Restart the inner loop continue; @@ -174,11 +185,21 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { *end_of_sequence = true; return Status::OK(); } + + // If we are not allowed to be sloppy and + // `worker_threads_[next_index]` has finished, advance `next_index`. + if (!dataset()->sloppy_ && worker_threads_[next_index_].finished) { + next_index_ = (next_index_ + 1) % num_workers; + continue; + } + // No values available; wait until woken up. + // TODO(jsimsa): Use slot-specific condition variable for + // coordination of elements consumption. cond_var_.wait(l); } return errors::Cancelled( - "SloppyInterleaveDatasetOp::Dataset::Iterator::GetNext"); + "ParallelInterleaveDatasetOp::Dataset::Iterator::GetNext"); } private: @@ -201,6 +222,16 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { condition_variable cond_var; }; + struct ThreadStatus { + // The underlying thread uses `finished` to communicate to the producer + // that it has finished. + bool finished = false; + // The underlying thread object. + std::unique_ptr thread; + + explicit ThreadStatus(Thread* thread) : thread(thread) {} + }; + Status EnsureWorkerThreadsStarted(IteratorContext* ctx) EXCLUSIVE_LOCKS_REQUIRED(mu_) { if (worker_threads_.empty()) { @@ -220,11 +251,10 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { std::unique_ptr itr; TF_RETURN_IF_ERROR(dataset::MakeIteratorFromInputElement( ctx, args, i, dataset()->captured_func_.get(), prefix(), &itr)); - worker_threads_.emplace_back( - std::unique_ptr(ctx->env()->StartThread( - {}, "worker_thread", - std::bind(&Iterator::WorkerThread, this, - new IteratorContext(*ctx), i, itr.release())))); + worker_threads_.emplace_back(ctx->env()->StartThread( + {}, "worker_thread", + std::bind(&Iterator::WorkerThread, this, + new IteratorContext(*ctx), i, itr.release()))); num_active_threads_ = i + 1; } } @@ -264,6 +294,7 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { std::unique_ptr out_iterator(out_iterator_ptr); auto cleanup = gtl::MakeCleanup([this, thread_index] { mutex_lock l(mu_); + worker_threads_[thread_index].finished = true; num_active_threads_--; cond_var_.notify_all(); }); @@ -345,13 +376,14 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { // Pointers to the worker threads. This must be last to ensure the // threads have exited before any other members are deallocated. // TODO(b/65178177): Avoid allocating additional threads. - std::vector> worker_threads_ GUARDED_BY(mu_); + std::vector worker_threads_ GUARDED_BY(mu_); }; const DatasetBase* const input_; const std::unique_ptr captured_func_; const int64 cycle_length_; const int64 block_length_; + const bool sloppy_; const DataTypeVector output_types_; const std::vector output_shapes_; }; @@ -362,8 +394,8 @@ class SloppyInterleaveDatasetOp : public UnaryDatasetOpKernel { NameAttrList func_; }; -REGISTER_KERNEL_BUILDER(Name("SloppyInterleaveDataset").Device(DEVICE_CPU), - SloppyInterleaveDatasetOp); +REGISTER_KERNEL_BUILDER(Name("ParallelInterleaveDataset").Device(DEVICE_CPU), + ParallelInterleaveDatasetOp); } // namespace diff --git a/tensorflow/core/kernels/parse_tensor_op.cc b/tensorflow/core/kernels/parse_tensor_op.cc index ab91a6ef677a95a498df1c3de85c8ea07d6451e8..6b599612ad7fde0bac44282521be26581aa752b8 100644 --- a/tensorflow/core/kernels/parse_tensor_op.cc +++ b/tensorflow/core/kernels/parse_tensor_op.cc @@ -92,6 +92,7 @@ class SerializeTensorOp : public OpKernel { Name("SerializeTensor").Device(DEVICE_CPU).TypeConstraint("T"), \ SerializeTensorOp); TF_CALL_ALL_TYPES(REGISTER) +TF_CALL_variant(REGISTER) #undef REGISTER } // namespace tensorflow diff --git a/tensorflow/core/kernels/prefetch_dataset_op.cc b/tensorflow/core/kernels/prefetch_dataset_op.cc index 8c846919c4b02e808b9369c2bc49a662482d0b21..a7aac508eb3f76a588f9fc39b761e33222a37041 100644 --- a/tensorflow/core/kernels/prefetch_dataset_op.cc +++ b/tensorflow/core/kernels/prefetch_dataset_op.cc @@ -59,7 +59,6 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel { Dataset(const DatasetBase* input, int64 buffer_size, IteratorContext::Params ctx_params) : input_(input), - buffer_size_(buffer_size), ctx_params_(std::move(ctx_params)) { input_->Ref(); diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/qr_op_impl.h index e263eb22f1f1e603a8374d0c4c9ea50a124c2ed3..0552c034d26ab7928c3141d1a3261bb486009a31 100644 --- a/tensorflow/core/kernels/qr_op_impl.h +++ b/tensorflow/core/kernels/qr_op_impl.h @@ -40,6 +40,7 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/cwise_ops.h" +#include "tensorflow/core/kernels/eye_functor.h" #include "tensorflow/core/kernels/matrix_band_part_op.h" #include "tensorflow/core/kernels/transpose_functor.h" #endif @@ -190,12 +191,9 @@ class QrOpGpu : public AsyncOpKernel { // Transpose input, since cuSolver uses column-major, while TensorFlow uses // row-major storage. - std::vector perm(ndims); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[ndims - 2], perm[ndims - 1]); const GPUDevice& device = context->eigen_device(); OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input, perm, &input_transposed), done); + context, DoMatrixTranspose(device, input, &input_transposed), done); // Compute QR decomposition in-place in input_transposed. std::vector dev_info; @@ -218,7 +216,7 @@ class QrOpGpu : public AsyncOpKernel { // and copy it to the output buffer. if (full_matrices_ || m == n) { OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input_transposed, perm, r), done); + context, DoMatrixTranspose(device, input_transposed, r), done); } else { const Scalar alpha(1); const Scalar beta(0); @@ -280,7 +278,7 @@ class QrOpGpu : public AsyncOpKernel { done); } OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input_transposed, perm, q), done); + context, DoMatrixTranspose(device, input_transposed, q), done); } // Asynchronously check return status from cuSolver kernels. diff --git a/tensorflow/core/kernels/quantize_op.cc b/tensorflow/core/kernels/quantize_op.cc index fd34e13c2929efda4a0101b06a8d904496fb7db7..fc26813a08e5afabf893f963eefa59fba18873d9 100644 --- a/tensorflow/core/kernels/quantize_op.cc +++ b/tensorflow/core/kernels/quantize_op.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/type_traits.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/cwise_ops.h" #include "tensorflow/core/kernels/meta_support.h" #include "tensorflow/core/kernels/quantization_utils.h" #include "tensorflow/core/lib/core/errors.h" @@ -31,6 +32,19 @@ enum { QUANTIZE_MODE_MIN_FIRST, QUANTIZE_MODE_SCALED, }; +enum { + // Round half away from zero: if the fraction of y is exactly 0.5, then + // round(y) = y + 0.5 if y > 0 + // round(y) = y - 0.5 if y < 0 + // E.g., -5.5 gets rounded to -6, -5.4 goes to -5, + // 5.4 goes to 5, and 5.5 goes to 6. + ROUND_HALF_AWAY_FROM_ZERO, + // Round half to even: if the fraction of y is exactly 0.5, then round(y) is + // the nearest even integer to y. + // E.g., 23.5 gets rounded to 24, 24.5 gets rounded to 24, while -23.5 becomes + // -24, and -24.5 gets rounded to 24. + ROUND_HALF_TO_EVEN, +}; } // namespace namespace tensorflow { @@ -66,6 +80,26 @@ class QuantizeV2Op : public OpKernel { } else if (mode_string == "SCALED") { mode_ = QUANTIZE_MODE_SCALED; } + + string round_mode_string; + OP_REQUIRES_OK(ctx, ctx->GetAttr("round_mode", &round_mode_string)); + OP_REQUIRES(ctx, + (round_mode_string == "HALF_AWAY_FROM_ZERO" || + round_mode_string == "HALF_TO_EVEN"), + errors::InvalidArgument("Round mode string must be " + "'HALF_AWAY_FROM_ZERO' or " + "'HALF_TO_EVEN', is '" + + round_mode_string + "'")); + if (round_mode_string == "HALF_AWAY_FROM_ZERO") { + round_mode_ = ROUND_HALF_AWAY_FROM_ZERO; + } else if (round_mode_string == "HALF_TO_EVEN") { + OP_REQUIRES(ctx, mode_string == "SCALED", + errors::InvalidArgument("Round mode 'HALF_TO_EVEN' " + "only supported for mode 'SCALED', " + "but mode is '" + + mode_string + "'.")); + round_mode_ = ROUND_HALF_TO_EVEN; + } } void Compute(OpKernelContext* ctx) override { @@ -151,40 +185,37 @@ class QuantizeV2Op : public OpKernel { typename TTypes::Vec o = output->template flat(); static constexpr int num_bits = sizeof(T) * 8; const float max_abs = std::max(std::abs(min_range), std::abs(max_range)); - bool is_signed = std::is_signed::value; + const bool is_signed = std::is_signed::value; + float target_range; if (is_signed) { max_range = max_abs; min_range = -max_abs; // If it is signed, we try to keep 0.0 being 0 and drop one bucket. For // example, if it is 8 bits, we have the range [-127, 127]. So for input // range of [-x, x], the scale should be 254/(2*x). - const float target_range = - static_cast((uint64_t{1} << (num_bits - 1)) - 1); - const float scale_factor = target_range / max_abs; - // Note that std::round is used to round the number before the cast. - // std::round implements "round-half-away-zero", - // e.g., -5.5 gets rounded to -6, -5.4 goes to -5, 5.4 goes to 5, - // and 5.5 goes to 6. - o.device(ctx->template eigen_device()) = - (input.flat().cwiseMin(max_range).cwiseMax(min_range) * - scale_factor) - .round() - .template cast(); + target_range = static_cast((uint64_t{1} << (num_bits - 1)) - 1); } else { max_range = max_abs; min_range = 0.0; // If it is unsigned and num_bits == 8, the range with 8 bits is [0, // 255]. If the input range is [0, x], then the scale is x/255 instead // of 254 as in the case above. - const float target_range = - static_cast((uint64_t{1} << num_bits) - 1); - const float scale_factor = target_range / max_abs; - // Because input is unsigned, we don't need to implement "round away - // from zero". The fast path avoids unaryExpr. + target_range = static_cast((uint64_t{1} << num_bits) - 1); + } + const float scale_factor = target_range / max_abs; + if (round_mode_ == ROUND_HALF_TO_EVEN) { + // scalar_round_op_google implements "round-half-to-even". o.device(ctx->template eigen_device()) = (input.flat().cwiseMin(max_range).cwiseMax(min_range) * - scale_factor + - 0.5f) + scale_factor) + .unaryExpr(Eigen::internal::scalar_round_op_google()) + .template cast(); + } else if (round_mode_ == ROUND_HALF_AWAY_FROM_ZERO) { + // scalar_round_op implements "round-half-away-from-zero". + o.device(ctx->template eigen_device()) = + (input.flat().cwiseMin(max_range).cwiseMax(min_range) * + scale_factor) + .unaryExpr(Eigen::internal::scalar_round_op()) .template cast(); } } @@ -201,6 +232,7 @@ class QuantizeV2Op : public OpKernel { private: float half_range_; int mode_; + int round_mode_; }; REGISTER_KERNEL_BUILDER( @@ -218,5 +250,4 @@ REGISTER_KERNEL_BUILDER( REGISTER_KERNEL_BUILDER( Name("QuantizeV2").Device(DEVICE_CPU).TypeConstraint("T"), QuantizeV2Op); - } // namespace tensorflow diff --git a/tensorflow/core/kernels/quantize_op_test.cc b/tensorflow/core/kernels/quantize_op_test.cc index 8a370966b44b6f6d817ab95807b7597cd2b3f7a0..d2cc55a94ddd7b3e31a5cfc841de25519abe2746 100644 --- a/tensorflow/core/kernels/quantize_op_test.cc +++ b/tensorflow/core/kernels/quantize_op_test.cc @@ -82,6 +82,7 @@ TEST_F(QuantizedOpTest, QuantizeV2Quint8Scaled) { test::FillValues(&expected_output_max, {255.0}); test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); } + TEST_F(QuantizedOpTest, QuantizeV2Quint8ScaledSmallInputRange) { TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") .Input(FakeInput(DT_FLOAT)) @@ -170,6 +171,66 @@ TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledSmallInputRange) { test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); } +TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledRoundToEven) { + TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("T", DataTypeToEnum::v()) + .Attr("mode", "SCALED") + .Attr("round_mode", "HALF_TO_EVEN") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({7}), + {-126.5, 0.0, 1.0, 2.5, 3.5, 64.0, 127.0}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_QINT8, TensorShape({7})); + // Input element 0.0 should map to 0. + // Input element 127.0 maps to 127. + test::FillValues(&expected, {-126, 0, 1, 2, 4, 64, 127}); + test::ExpectTensorEqual(expected, *GetOutput(0)); + + Tensor expected_output_min(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_min, {-127.0}); + test::ExpectTensorEqual(expected_output_min, *GetOutput(1)); + + Tensor expected_output_max(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_max, {127.0}); + test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); +} + +TEST_F(QuantizedOpTest, QuantizeV2Qint8ScaledRoundAwayFromZero) { + TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Input(FakeInput(DT_FLOAT)) + .Attr("T", DataTypeToEnum::v()) + .Attr("mode", "SCALED") + .Attr("round_mode", "HALF_AWAY_FROM_ZERO") + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + AddInputFromArray(TensorShape({7}), + {-126.5, 0.0, 1.0, 2.5, 3.5, 64.0, 127.0}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + AddInputFromArray(TensorShape({1}), {-127.0f}); + TF_ASSERT_OK(RunOpKernel()); + Tensor expected(allocator(), DT_QINT8, TensorShape({7})); + // Input element 0.0 should map to 0. + // Input element 127.0 maps to 127. + test::FillValues(&expected, {-127, 0, 1, 3, 4, 64, 127}); + test::ExpectTensorEqual(expected, *GetOutput(0)); + + Tensor expected_output_min(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_min, {-127.0}); + test::ExpectTensorEqual(expected_output_min, *GetOutput(1)); + + Tensor expected_output_max(allocator(), DT_FLOAT, TensorShape({})); + test::FillValues(&expected_output_max, {127.0}); + test::ExpectTensorEqual(expected_output_max, *GetOutput(2)); +} + TEST_F(QuantizedOpTest, QuantizeV2_32Bit) { TF_ASSERT_OK(NodeDefBuilder("quantize_op", "QuantizeV2") .Input(FakeInput(DT_FLOAT)) diff --git a/tensorflow/core/kernels/range_dataset_op.cc b/tensorflow/core/kernels/range_dataset_op.cc index a57c21a590b453b362d7700e79d531cd5958c650..7adfcc4f8d29c67007ae08a621fd5bef0eddd498 100644 --- a/tensorflow/core/kernels/range_dataset_op.cc +++ b/tensorflow/core/kernels/range_dataset_op.cc @@ -112,19 +112,16 @@ class RangeDatasetOp : public DatasetOpKernel { } protected: - Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) override { + Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("next"), next_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("next"), next_)); return Status::OK(); } Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) override { + IteratorStateReader* reader) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("next"), &next_)); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("next"), &next_)); return Status::OK(); } diff --git a/tensorflow/core/kernels/reader_dataset_ops.cc b/tensorflow/core/kernels/reader_dataset_ops.cc index b455c28e07cb0c32b5d3d9837398e6faa074027c..39ef92a5dec0def5ae51e41feac38f1257693376 100644 --- a/tensorflow/core/kernels/reader_dataset_ops.cc +++ b/tensorflow/core/kernels/reader_dataset_ops.cc @@ -54,14 +54,9 @@ class TextLineDatasetOp : public DatasetOpKernel { io::ZlibCompressionOptions zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); - bool use_compression = false; - if (compression_type.empty()) { - use_compression = false; - } else if (compression_type == "ZLIB") { - use_compression = true; + if (compression_type == "ZLIB") { zlib_compression_options = io::ZlibCompressionOptions::DEFAULT(); } else if (compression_type == "GZIP") { - use_compression = true; zlib_compression_options = io::ZlibCompressionOptions::GZIP(); } else { OP_REQUIRES(ctx, compression_type.empty(), @@ -79,17 +74,20 @@ class TextLineDatasetOp : public DatasetOpKernel { filenames.push_back(filenames_tensor->flat()(i)); } - *output = new Dataset(std::move(filenames), use_compression, + *output = new Dataset(ctx, std::move(filenames), compression_type, zlib_compression_options); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - Dataset(std::vector filenames, bool use_compression, + Dataset(OpKernelContext* ctx, std::vector filenames, + const string& compression_type, const io::ZlibCompressionOptions& options) - : filenames_(std::move(filenames)), - use_compression_(use_compression), + : GraphDatasetBase(ctx), + filenames_(std::move(filenames)), + compression_type_(compression_type), + use_compression_(!compression_type.empty()), options_(options) {} std::unique_ptr MakeIterator( @@ -111,6 +109,21 @@ class TextLineDatasetOp : public DatasetOpKernel { string DebugString() override { return "TextLineDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + Node* filenames = nullptr; + Node* compression_type = nullptr; + Node* buffer_size = nullptr; + TF_RETURN_IF_ERROR(b->AddVector(filenames_, &filenames)); + TF_RETURN_IF_ERROR(b->AddScalar(compression_type_, &compression_type)); + TF_RETURN_IF_ERROR( + b->AddScalar(options_.input_buffer_size, &buffer_size)); + TF_RETURN_IF_ERROR(b->AddDataset( + this, {filenames, compression_type, buffer_size}, output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -123,7 +136,7 @@ class TextLineDatasetOp : public DatasetOpKernel { mutex_lock l(mu_); do { // We are currently processing a file, so try to read the next line. - if (processing_file_) { + if (buffered_input_stream_) { string line_contents; Status s = buffered_input_stream_->ReadLine(&line_contents); @@ -138,14 +151,9 @@ class TextLineDatasetOp : public DatasetOpKernel { // Report non-EOF errors to the caller. return s; } - // We have reached the end of the current file, so maybe // move on to next file. - processing_file_ = false; - input_stream_.reset(); - zlib_input_stream_.reset(); - buffered_input_stream_.reset(); - file_.reset(); + ResetStreamsLocked(); ++current_file_index_; } @@ -155,30 +163,86 @@ class TextLineDatasetOp : public DatasetOpKernel { return Status::OK(); } - // Actually move on to next file. - TF_RETURN_IF_ERROR(ctx->env()->NewRandomAccessFile( - dataset()->filenames_[current_file_index_], &file_)); - processing_file_ = true; - input_stream_.reset( - new io::RandomAccessInputStream(file_.get(), false)); - if (dataset()->use_compression_) { - zlib_input_stream_.reset(new io::ZlibInputStream( - input_stream_.get(), dataset()->options_.input_buffer_size, - dataset()->options_.input_buffer_size, dataset()->options_)); - buffered_input_stream_.reset(new io::BufferedInputStream( - zlib_input_stream_.get(), dataset()->options_.input_buffer_size, - false)); - } else { - buffered_input_stream_.reset(new io::BufferedInputStream( - input_stream_.get(), dataset()->options_.input_buffer_size, - false)); - } + TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); } while (true); } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"), + current_file_index_)); + + // `buffered_input_stream_` is empty if + // 1. GetNext has not been called even once. + // 2. All files have been read and iterator has been exhausted. + if (buffered_input_stream_) { + TF_RETURN_IF_ERROR(writer->WriteScalar( + full_name("current_pos"), buffered_input_stream_->Tell())); + } + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + ResetStreamsLocked(); + int64 current_file_index; + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"), + ¤t_file_index)); + current_file_index_ = size_t(current_file_index); + // The key "current_pos" is written only if the iterator was saved + // with an open file. + if (reader->Contains(full_name("current_pos"))) { + int64 current_pos; + TF_RETURN_IF_ERROR( + reader->ReadScalar(full_name("current_pos"), ¤t_pos)); + + TF_RETURN_IF_ERROR(SetupStreamsLocked(ctx->env())); + TF_RETURN_IF_ERROR(buffered_input_stream_->Seek(current_pos)); + } + return Status::OK(); + } + private: + // Sets up reader streams to read from the file at `current_file_index_`. + Status SetupStreamsLocked(Env* env) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (current_file_index_ >= dataset()->filenames_.size()) { + return errors::InvalidArgument( + "current_file_index_:", current_file_index_, + " >= filenames_.size():", dataset()->filenames_.size()); + } + + // Actually move on to next file. + TF_RETURN_IF_ERROR(env->NewRandomAccessFile( + dataset()->filenames_[current_file_index_], &file_)); + input_stream_.reset( + new io::RandomAccessInputStream(file_.get(), false)); + + if (dataset()->use_compression_) { + zlib_input_stream_.reset(new io::ZlibInputStream( + input_stream_.get(), dataset()->options_.input_buffer_size, + dataset()->options_.input_buffer_size, dataset()->options_)); + buffered_input_stream_.reset(new io::BufferedInputStream( + zlib_input_stream_.get(), dataset()->options_.input_buffer_size, + false)); + } else { + buffered_input_stream_.reset(new io::BufferedInputStream( + input_stream_.get(), dataset()->options_.input_buffer_size, + false)); + } + return Status::OK(); + } + + // Resets all reader streams. + void ResetStreamsLocked() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + input_stream_.reset(); + zlib_input_stream_.reset(); + buffered_input_stream_.reset(); + file_.reset(); + } + mutex mu_; - bool processing_file_ GUARDED_BY(mu_) = false; std::unique_ptr input_stream_ GUARDED_BY(mu_); std::unique_ptr zlib_input_stream_ GUARDED_BY(mu_); @@ -190,6 +254,7 @@ class TextLineDatasetOp : public DatasetOpKernel { }; const std::vector filenames_; + const string compression_type_; const bool use_compression_; const io::ZlibCompressionOptions options_; }; @@ -356,31 +421,30 @@ class FixedLengthRecordDatasetOp : public DatasetOpKernel { } protected: - Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) override { + Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar( - full_name("current_file_index"), current_file_index_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("current_file_index"), + current_file_index_)); // `input_buffer_` is empty if // 1. GetNext has not been called even once. // 2. All files have been read and iterator has been exhausted. int64 current_pos = input_buffer_ ? input_buffer_->Tell() : -1; TF_RETURN_IF_ERROR( - writer->WriteScalar(full_name("current_pos"), current_pos)); + writer->WriteScalar(full_name("current_pos"), current_pos)); return Status::OK(); } Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) override { + IteratorStateReader* reader) override { mutex_lock l(mu_); int64 current_file_index; - TF_RETURN_IF_ERROR(reader->ReadScalar( - full_name("current_file_index"), ¤t_file_index)); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("current_file_index"), + ¤t_file_index)); current_file_index_ = size_t(current_file_index); int64 current_pos; TF_RETURN_IF_ERROR( - reader->ReadScalar(full_name("current_pos"), ¤t_pos)); + reader->ReadScalar(full_name("current_pos"), ¤t_pos)); // Seek to current_pos. input_buffer_.reset(); diff --git a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h index be9a61188198a0ab24150cda909a348867914bcd..36ca7f834f7b4fe7db1e2591189b1359231c7307 100644 --- a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h +++ b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h @@ -460,7 +460,7 @@ void LaunchScalarReduction(OpKernelContext* ctx, OUT_T out, IN_T in, return; } else if (in_size <= 1 << 19) { const int num_threads = 256; - const int num_blocks = min(32, Eigen::divup(in_size, num_threads)); + const int num_blocks = std::min(32, Eigen::divup(in_size, num_threads)); // it seems like tailoring this to the GPU // would be more effective, but all attempts // at making this a multiple of the number of @@ -557,13 +557,13 @@ void LaunchColumnReduction_LTE16Cols(OpKernelContext* ctx, OUT_T out, IN_T in, int extent_x, int extent_y, Op op, T init, const cudaStream_t& cu_stream) { int rows_per_warp = 32 / extent_y; - dim3 block_dim(32, min(Eigen::divup(extent_x, rows_per_warp), 32), 1); + dim3 block_dim(32, std::min(Eigen::divup(extent_x, rows_per_warp), 32), 1); dim3 grid_dim(1, Eigen::divup(static_cast(extent_x), rows_per_warp * block_dim.y), 1); - grid_dim.y = min((int)grid_dim.y, 32); + grid_dim.y = std::min((int)grid_dim.y, 32); if (grid_dim.y > 2 && grid_dim.y < 32) { int log2 = Log2Floor(grid_dim.y); @@ -596,10 +596,10 @@ template void LaunchColumnReduction_LTE4096Cols(OpKernelContext* ctx, OUT_T out, IN_T in, int extent_x, int extent_y, Op op, T init, const cudaStream_t& cu_stream) { - dim3 block_dim(32, min(extent_x, 32), 1); + dim3 block_dim(32, std::min(extent_x, 32), 1); dim3 grid_dim((extent_y + 31) / 32, 1, 1); - if (grid_dim.x < 16) grid_dim.y = min((extent_x + 31) / 32, 32); + if (grid_dim.x < 16) grid_dim.y = std::min((extent_x + 31) / 32, 32); if (grid_dim.y > 2 && grid_dim.y < 32) { int log2 = Log2Floor(grid_dim.y); diff --git a/tensorflow/core/kernels/reduction_ops_all.cc b/tensorflow/core/kernels/reduction_ops_all.cc index 41abc2b9574fa0f00932ae23bf47364d9de8ad97..4a34c4ef513bdd390cb8a0367d457ad19ba4a274 100644 --- a/tensorflow/core/kernels/reduction_ops_all.cc +++ b/tensorflow/core/kernels/reduction_ops_all.cc @@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_CPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("All") + .TypeConstraint("Tidx") + .Device(DEVICE_CPU) + .HostMemory("reduction_indices"), + ReductionOp); #if GOOGLE_CUDA REGISTER_KERNEL_BUILDER( @@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_GPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("All") + .TypeConstraint("Tidx") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices"), + ReductionOp); #endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_any.cc b/tensorflow/core/kernels/reduction_ops_any.cc index a2087cc3b7b6dc846c14a169a2209297713deb56..6c0519de95e7773e5082e42ad319bad9beab81d8 100644 --- a/tensorflow/core/kernels/reduction_ops_any.cc +++ b/tensorflow/core/kernels/reduction_ops_any.cc @@ -22,7 +22,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_CPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("Any") + .TypeConstraint("Tidx") + .Device(DEVICE_CPU) + .HostMemory("reduction_indices"), + ReductionOp); #if GOOGLE_CUDA REGISTER_KERNEL_BUILDER( @@ -30,7 +36,13 @@ REGISTER_KERNEL_BUILDER( .TypeConstraint("Tidx") .Device(DEVICE_GPU) .HostMemory("reduction_indices"), - ReductionOp); + ReductionOp); +REGISTER_KERNEL_BUILDER( + Name("Any") + .TypeConstraint("Tidx") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices"), + ReductionOp); #endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_common.cc b/tensorflow/core/kernels/reduction_ops_common.cc index 5eba4288acccfb2465e9c483e9fefda4adf68185..8daab0d6be499a38d57c4c6a0c96a38f37985407 100644 --- a/tensorflow/core/kernels/reduction_ops_common.cc +++ b/tensorflow/core/kernels/reduction_ops_common.cc @@ -57,13 +57,12 @@ gtl::InlinedVector ReductionHelper::permutation() { return perm; } -Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis, - const bool keep_dims) { - // bitmap[i] indicates whether to reduce data along i-th axis. - gtl::InlinedVector bitmap(data.dims(), false); - auto axis_vec = axis.flat(); +template +Status SimplifyHelper(const Tensor& data, const Tensor& axis, + gtl::InlinedVector& bitmap) { + auto axis_vec = axis.flat(); for (int64 i = 0; i < axis.NumElements(); ++i) { - int32 index = axis_vec(i); + Tperm index = axis_vec(i); if (index < -data.dims() || index >= data.dims()) { return errors::InvalidArgument("Invalid reduction dimension (", index, " for input with ", data.dims(), @@ -72,7 +71,18 @@ Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis, index = (index + data.dims()) % data.dims(); bitmap[index] = true; } + return Status::OK(); +} +Status ReductionHelper::Simplify(const Tensor& data, const Tensor& axis, + const bool keep_dims) { + // bitmap[i] indicates whether to reduce data along i-th axis. + gtl::InlinedVector bitmap(data.dims(), false); + if (axis.dtype() == DT_INT32) { + TF_RETURN_IF_ERROR(SimplifyHelper(data, axis, bitmap)); + } else { + TF_RETURN_IF_ERROR(SimplifyHelper(data, axis, bitmap)); + } // Output tensor's dim sizes. out_shape_.clear(); for (int i = 0; i < data.dims(); ++i) { diff --git a/tensorflow/core/kernels/reduction_ops_common.h b/tensorflow/core/kernels/reduction_ops_common.h index 71af9d88dc1d34db392cd1e29714bdcad645abd9..9da992ccd18d7bf107a1bc2a7b91ec9fb1a85fd5 100644 --- a/tensorflow/core/kernels/reduction_ops_common.h +++ b/tensorflow/core/kernels/reduction_ops_common.h @@ -25,6 +25,7 @@ limitations under the License. #include "third_party/eigen3/Eigen/Core" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -42,7 +43,7 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL template struct Constants { @@ -68,11 +69,13 @@ struct ConstantsBase { const Eigen::IndexList> kOne; const Eigen::IndexList, Eigen::type2index<2>> kZeroTwo; }; -template<> struct Constants : ConstantsBase{}; +template <> +struct Constants : ConstantsBase {}; #ifdef TENSORFLOW_USE_SYCL -template<> struct Constants : ConstantsBase{}; -#endif // TENSORFLOW_USE_SYCL -#endif // EIGEN_HAS_INDEX_LIST +template <> +struct Constants : ConstantsBase {}; +#endif // TENSORFLOW_USE_SYCL +#endif // EIGEN_HAS_INDEX_LIST class ReductionHelper { public: @@ -131,12 +134,13 @@ class ReductionHelper { // For operations where the output is a reduction function along some // dimensions of the input. -template +template class ReductionOp : public OpKernel { public: explicit ReductionOp(OpKernelConstruction* ctx) : OpKernel(ctx) { const DataType dt = DataTypeToEnum::v(); - OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, DT_INT32}, {dt})); + const DataType pt = DataTypeToEnum::v(); + OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, pt}, {dt})); OP_REQUIRES_OK(ctx, ctx->GetAttr("keep_dims", &keep_dims_)); } @@ -266,20 +270,19 @@ struct ReduceFunctorBase { } template - static void FillIdentity(const Device& d, OUT_T out, - const Reducer& reducer) { + static void FillIdentity(const Device& d, OUT_T out, const Reducer& reducer) { FillIdentityEigenImpl(d, out, reducer); } }; template struct ReduceFunctor - : ReduceFunctorBase{}; + : ReduceFunctorBase {}; #if TENSORFLOW_USE_SYCL template struct ReduceFunctor - : ReduceFunctorBase{}; -#endif // TENSORFLOW_USE_SYCL + : ReduceFunctorBase {}; +#endif // TENSORFLOW_USE_SYCL } // namespace functor } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_max.cc b/tensorflow/core/kernels/reduction_ops_max.cc index 4ca5c11a4858e86ae553fdd54d918fce18edd650..9cf953f4bfed9dabd27a2297cc0d3ccf4f98ff32 100644 --- a/tensorflow/core/kernels/reduction_ops_max.cc +++ b/tensorflow/core/kernels/reduction_ops_max.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Max") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Max") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Max") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_GPU_KERNELS(float); REGISTER_GPU_KERNELS(double); REGISTER_GPU_KERNELS(int64); @@ -52,21 +65,37 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Max") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_GPU_KERNELS #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Max") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Max") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Max") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); @@ -78,8 +107,17 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Max") + .Device(DEVICE_SYCL) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_mean.cc b/tensorflow/core/kernels/reduction_ops_mean.cc index 5b01de8ddbcdb90fc059aab9bd488cf80984a04a..f61589f913b14bd99bba8b8a43b01b0213b1ff17 100644 --- a/tensorflow/core/kernels/reduction_ops_mean.cc +++ b/tensorflow/core/kernels/reduction_ops_mean.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Mean") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Mean") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); @@ -45,17 +58,24 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Mean") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Mean") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc index 1e394bea41fb41149266b5a7fb3fe93b3d67b1a8..807ac0a4567790ef3fb95b4c12a91a1562f83fa7 100644 --- a/tensorflow/core/kernels/reduction_ops_min.cc +++ b/tensorflow/core/kernels/reduction_ops_min.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Min") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Min") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Min") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_GPU_KERNELS(float); REGISTER_GPU_KERNELS(double); @@ -51,21 +64,37 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Min") + .Device(DEVICE_GPU) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_GPU_KERNELS #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Min") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Min") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Min") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); @@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER( .HostMemory("output") .TypeConstraint("T") .TypeConstraint("Tidx"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Min") + .Device(DEVICE_SYCL) + .HostMemory("reduction_indices") + .HostMemory("input") + .HostMemory("output") + .TypeConstraint("T") + .TypeConstraint("Tidx"), + ReductionOp>); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_prod.cc b/tensorflow/core/kernels/reduction_ops_prod.cc index 33f6ae6bae11b2d8bee739fd7a1d4a8486756864..e9b23df74604da8e393676c7781c724586956342 100644 --- a/tensorflow/core/kernels/reduction_ops_prod.cc +++ b/tensorflow/core/kernels/reduction_ops_prod.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Prod") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Prod") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_int32(REGISTER_GPU_KERNELS); TF_CALL_complex64(REGISTER_GPU_KERNELS); @@ -46,18 +59,25 @@ TF_CALL_complex128(REGISTER_GPU_KERNELS); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Prod") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Prod") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(int32); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/reduction_ops_sum.cc b/tensorflow/core/kernels/reduction_ops_sum.cc index c1f4f3475af68f73e7b34ea4319db22f62f769e1..5318d8c1339eb5cd9429105082cb50e478d21c41 100644 --- a/tensorflow/core/kernels/reduction_ops_sum.cc +++ b/tensorflow/core/kernels/reduction_ops_sum.cc @@ -17,26 +17,39 @@ limitations under the License. namespace tensorflow { -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Sum") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ReductionOp>); +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ReductionOp>); TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Sum") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER( \ + Name("Sum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); @@ -53,19 +66,35 @@ REGISTER_KERNEL_BUILDER( .HostMemory("input") .HostMemory("output") .HostMemory("reduction_indices"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Sum") + .Device(DEVICE_GPU) + .TypeConstraint("T") + .TypeConstraint("Tidx") + .HostMemory("input") + .HostMemory("output") + .HostMemory("reduction_indices"), + ReductionOp>); #endif #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Sum") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("reduction_indices"), \ - ReductionOp>); +#define REGISTER_SYCL_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("Sum") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); \ + REGISTER_KERNEL_BUILDER(Name("Sum") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("reduction_indices"), \ + ReductionOp>); REGISTER_SYCL_KERNELS(float); REGISTER_SYCL_KERNELS(double); @@ -77,8 +106,17 @@ REGISTER_KERNEL_BUILDER( .HostMemory("input") .HostMemory("output") .HostMemory("reduction_indices"), - ReductionOp>); + ReductionOp>); +REGISTER_KERNEL_BUILDER( + Name("Sum") + .Device(DEVICE_SYCL) + .TypeConstraint("T") + .TypeConstraint("Tidx") + .HostMemory("input") + .HostMemory("output") + .HostMemory("reduction_indices"), + ReductionOp>); #undef REGISTER_SYCL_KERNELS -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/repeat_dataset_op.cc b/tensorflow/core/kernels/repeat_dataset_op.cc index 5d836927d220c2044c534cc257b138f0e6b7bb83..9813e99a70bc51e725a2974e759f3708d4f9b4d3 100644 --- a/tensorflow/core/kernels/repeat_dataset_op.cc +++ b/tensorflow/core/kernels/repeat_dataset_op.cc @@ -124,19 +124,18 @@ class RepeatDatasetOp : public UnaryDatasetOpKernel { } protected: - Status SaveInternal(OpKernelContext* ctx, - IteratorBundleWriter* writer) override { + Status SaveInternal(IteratorStateWriter* writer) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); - TF_RETURN_IF_ERROR(writer->SaveParent(ctx, input_impl_)); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); + TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_)); return Status::OK(); } Status RestoreInternal(OpKernelContext* ctx, - IteratorBundleReader* reader) override { + IteratorStateReader* reader) override { mutex_lock l(mu_); - TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); - TF_RETURN_IF_ERROR(reader->RestoreParent(ctx, input_impl_)); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); + TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_)); return Status::OK(); } diff --git a/tensorflow/core/kernels/resize_bicubic_op.cc b/tensorflow/core/kernels/resize_bicubic_op.cc index 1c43e77e7c2ea241b97b9ccd8fe791a7867ece64..1a9cf4c6406d85bf26b43e0b9b855760a4888a4c 100644 --- a/tensorflow/core/kernels/resize_bicubic_op.cc +++ b/tensorflow/core/kernels/resize_bicubic_op.cc @@ -20,7 +20,6 @@ limitations under the License. #include #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" @@ -29,6 +28,7 @@ limitations under the License. #include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" namespace tensorflow { namespace { @@ -235,6 +235,7 @@ inline void interpolate_with_caching( const T* input_b_ptr = input_data.data(); float* output_y_ptr = output_data.data(); + std::vector cached_value(num_channels == 3 ? 0 : 4 * num_channels, 0); for (int64 b = 0; b < resizer_state.batch_size; ++b, input_b_ptr += in_batch_width) { @@ -248,6 +249,7 @@ inline void interpolate_with_caching( const T* y_ptr_1 = input_b_ptr + y_wai.index_1 * in_row_width; const T* y_ptr_2 = input_b_ptr + y_wai.index_2 * in_row_width; const T* y_ptr_3 = input_b_ptr + y_wai.index_3 * in_row_width; + if (num_channels == 3) { // Manually unroll case of 3 channels. float cached_value_0[4] = {0}; @@ -330,48 +332,61 @@ inline void interpolate_with_caching( x_wai.weight_2, x_wai.weight_3); } } else { - for (int64 c = 0; c < num_channels; ++c) { - float cached_value[4] = {0}; - for (int64 x = 0; x < resizer_state.out_width; ++x) { - const WeightsAndIndices& x_wai = x_wais[x]; - // Shift values in cached_value to fill first 'advance' values. - switch (x_wai.advance) { - case 3: - cached_value[0] = cached_value[1]; - cached_value[1] = cached_value[2]; - cached_value[2] = cached_value[3]; - break; - case 2: - cached_value[0] = cached_value[2]; - cached_value[1] = cached_value[3]; - break; - case 1: { - cached_value[0] = cached_value[3]; - break; + for (int64 x = 0; x < resizer_state.out_width; ++x) { + const WeightsAndIndices& x_wai = x_wais[x]; + // Shift values in cached_value to fill first 'advance' values. + switch (x_wai.advance) { + case 3: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = cached_value[4 * c + 1]; + cached_value[4 * c + 1] = cached_value[4 * c + 2]; + cached_value[4 * c + 2] = cached_value[4 * c + 3]; + } + break; + case 2: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = cached_value[4 * c + 2]; + cached_value[4 * c + 1] = cached_value[4 * c + 3]; + } + break; + case 1: { + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = cached_value[4 * c + 3]; } + break; } + } - // Set the remaining '4-advance' values by computing. - switch (x_wai.advance) { - case 0: - cached_value[0] = ComputeYInterpolation( + // Set the remaining '4-advance' values by computing. + switch (x_wai.advance) { + case 0: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 0] = ComputeYInterpolation( 0, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - TF_FALLTHROUGH_INTENDED; - case 1: - cached_value[1] = ComputeYInterpolation( + } + TF_FALLTHROUGH_INTENDED; + case 1: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 1] = ComputeYInterpolation( 1, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - TF_FALLTHROUGH_INTENDED; - case 2: - cached_value[2] = ComputeYInterpolation( + } + TF_FALLTHROUGH_INTENDED; + case 2: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 2] = ComputeYInterpolation( 2, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - TF_FALLTHROUGH_INTENDED; - case 3: - cached_value[3] = ComputeYInterpolation( + } + TF_FALLTHROUGH_INTENDED; + case 3: + for (int64 c = 0; c < num_channels; ++c) { + cached_value[4 * c + 3] = ComputeYInterpolation( 3, c, y_wai, y_ptr_0, y_ptr_1, y_ptr_2, y_ptr_3, x_wai); - break; - } + } + break; + } + for (int64 c = 0; c < num_channels; ++c) { output_y_ptr[x * num_channels + c] = - Compute(cached_value, x_wai.weight_0, x_wai.weight_1, + Compute(&cached_value[4 * c], x_wai.weight_0, x_wai.weight_1, x_wai.weight_2, x_wai.weight_3); } } diff --git a/tensorflow/core/kernels/resize_bicubic_op_test.cc b/tensorflow/core/kernels/resize_bicubic_op_test.cc index ae14d2804e2bfbb61d71ce0ab4026a2b19293beb..9e10fec42321023d95f3ae8d32a5a1c8f2c7a94e 100644 --- a/tensorflow/core/kernels/resize_bicubic_op_test.cc +++ b/tensorflow/core/kernels/resize_bicubic_op_test.cc @@ -251,14 +251,15 @@ TEST_F(ResizeBicubicOpTest, TestAreaRandomDataSeveralInputsSizes4Channels) { RunManyRandomTests(4); } -static Graph* ResizeBicubic(int batch_size, int size, int channels) { +static Graph* ResizeBicubic(int batch_size, int size, int channels, + float scale_y = 0.3, float scale_x = 0.7) { Graph* g = new Graph(OpRegistry::Global()); Tensor input(DT_FLOAT, TensorShape({batch_size, size, size, channels})); input.flat().setRandom(); Tensor shape(DT_INT32, TensorShape({2})); auto shape_t = shape.flat(); - shape_t(0) = 0.3 * size; - shape_t(1) = 0.7 * size; + shape_t(0) = scale_y * size; + shape_t(1) = scale_x * size; test::graph::Binary(g, "ResizeBicubic", test::graph::Constant(g, input), test::graph::Constant(g, shape)); return g; @@ -285,4 +286,17 @@ BM_ResizeBicubicDev(32, 128, 3); BM_ResizeBicubicDev(32, 512, 3); BM_ResizeBicubicDev(32, 1024, 3); +#define BM_ResizeBicubicExpand(BATCH, SIZE, CHANNELS) \ + static void BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS(int iters) { \ + testing::ItemsProcessed(static_cast(iters) * BATCH * SIZE * SIZE * \ + CHANNELS * 8 * 8); \ + test::Benchmark("cpu", ResizeBicubic(BATCH, SIZE, CHANNELS, 8, 8)) \ + .Run(iters); \ + } \ + BENCHMARK(BM_ResizeBicubicExpand##_##BATCH##_##SIZE##_##CHANNELS); + +BM_ResizeBicubicExpand(12, 48, 1); +BM_ResizeBicubicExpand(12, 48, 3); +BM_ResizeBicubicExpand(12, 48, 40); + } // end namespace tensorflow diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc index 3cca4939729a4ec174f4911e7acdddf6cf118f52..0ae8a8fdbc14af81650fb756fdd20bb0d983e71e 100644 --- a/tensorflow/core/kernels/resource_variable_ops.cc +++ b/tensorflow/core/kernels/resource_variable_ops.cc @@ -85,7 +85,7 @@ class ReadVariableOp : public OpKernel { errors::NotFound( "Error while reading resource variable ", handle.name(), " from Container: ", handle.container(), - ". This could mean that the variable was not initialized. ", + ". This could mean that the variable was uninitialized. ", status.ToString())); core::ScopedUnref s(variable); @@ -200,6 +200,9 @@ class DestroyResourceOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("DestroyResourceOp").Device(DEVICE_CPU), DestroyResourceOp); +REGISTER_KERNEL_BUILDER( + Name("DestroyResourceOp").Device(DEVICE_GPU).HostMemory("resource"), + DestroyResourceOp); template class AssignVariableOp : public OpKernel { @@ -461,7 +464,7 @@ class ResourceGatherOp : public OpKernel { auto out_flat = out->shaped({1, N, out->NumElements() / N}); functor::GatherFunctor functor; - int64 bad_i = functor(c->eigen_device(), params_flat, + int64 bad_i = functor(c, params_flat, indices_flat, out_flat); OP_REQUIRES( @@ -566,9 +569,11 @@ class ResourceScatterUpdateOp : public OpKernel { REGISTER_SCATTER_KERNEL_INDEX(type, int64, dev, name, op); // TODO(apassos) add the other types here. -#define REGISTER_SCATTER_ARITHEMTIC(type, dev) \ - REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterAdd", \ - scatter_op::UpdateOp::ADD); +#define REGISTER_SCATTER_ARITHEMTIC(type, dev) \ + REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterAdd", \ + scatter_op::UpdateOp::ADD); \ + REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterUpdate", \ + scatter_op::UpdateOp::ASSIGN); // Registers CPU kernels. #define REGISTER_SCATTER_ARITHEMTIC_CPU(type) \ diff --git a/tensorflow/core/kernels/scan_ops.cc b/tensorflow/core/kernels/scan_ops.cc index cc434ab0aee11aead575c21fa800267baec2dd51..0a6848361a05559e8d1e23318ca66a9dd3ad9a95 100644 --- a/tensorflow/core/kernels/scan_ops.cc +++ b/tensorflow/core/kernels/scan_ops.cc @@ -35,7 +35,7 @@ namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; -template +template class ScanOp : public OpKernel { public: explicit ScanOp(OpKernelConstruction* ctx) : OpKernel(ctx) { @@ -51,8 +51,9 @@ class ScanOp : public OpKernel { errors::InvalidArgument("ScanOp: axis must be a scalar, not ", tensor_axis.shape().DebugString())); - const int axis_arg = internal::SubtleMustCopy(tensor_axis.scalar()()); - const int axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg; + const Tidx axis_arg = + internal::SubtleMustCopy(tensor_axis.scalar()()); + const Tidx axis = (axis_arg < 0) ? input.dims() + axis_arg : axis_arg; OP_REQUIRES(ctx, FastBoundsCheck(axis, input.dims()), errors::InvalidArgument( "ScanOp: Expected scan axis in the range [", -input.dims(), @@ -70,11 +71,11 @@ class ScanOp : public OpKernel { // Dim reduction. int64 reduced_shape[3] = {1, 1, 1}; - for (int i = 0; i < axis; ++i) { + for (Tidx i = 0; i < axis; ++i) { reduced_shape[0] *= input.dim_size(i); } reduced_shape[1] = input.dim_size(axis); - for (int i = axis + 1; i < input.dims(); ++i) { + for (Tidx i = axis + 1; i < input.dims(); ++i) { reduced_shape[2] *= input.dim_size(i); } @@ -112,51 +113,76 @@ TF_CALL_GPU_NUMBER_TYPES(DECLARE_FOR_ALL_REDUCERS); } // namespace functor #endif // GOOGLE_CUDA - // Register Cumsum kernels -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumsum") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ScanOp>) +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int64>) TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumsum") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("axis"), \ - ScanOp>) +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumsum") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int64>) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS) #undef REGISTER_GPU_KERNELS #endif // GOOGLE_CUDA // Register Cumprod kernels -#define REGISTER_CPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumprod") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx"), \ - ScanOp>) +#define REGISTER_CPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx"), \ + ScanOp, int64>) TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS #if GOOGLE_CUDA -#define REGISTER_GPU_KERNELS(type) \ - REGISTER_KERNEL_BUILDER( \ - Name("Cumprod") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("axis"), \ - ScanOp>) +#define REGISTER_GPU_KERNELS(type) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int32>) \ + REGISTER_KERNEL_BUILDER( \ + Name("Cumprod") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("axis"), \ + ScanOp, int64>) TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS) #undef REGISTER_GPU_KERNELS #endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc b/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc index b0b4f89a27563fdd4b4c07095ad48e2c43976aad..3a84df07a9ab9ad06fe9a97d48475492880eb810 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc +++ b/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc @@ -148,11 +148,8 @@ class SelfAdjointEigV2OpGpu : public AsyncOpKernel { if (compute_v_) { // Transpose eigenvectors now stored in input_copy in column-major form to // output in row-major form. - std::vector perm(ndims); - std::iota(perm.begin(), perm.end(), 0); - std::swap(perm[ndims - 2], perm[ndims - 1]); OP_REQUIRES_OK_ASYNC( - context, DoTranspose(device, input_copy, perm, eigenvectors), done); + context, DoMatrixTranspose(device, input_copy, eigenvectors), done); } // Asynchronously check return status from cuSolver kernels. diff --git a/tensorflow/core/kernels/sequence_ops.cc b/tensorflow/core/kernels/sequence_ops.cc index c8ea92302010dca26e66c2536478952cce4681ca..e2e3758d87e49702ebc48f78c022affe49a3b7e4 100644 --- a/tensorflow/core/kernels/sequence_ops.cc +++ b/tensorflow/core/kernels/sequence_ops.cc @@ -96,7 +96,7 @@ TF_CALL_double(REGISTER_SYCL_KERNEL); TF_CALL_int32(REGISTER_SYCL_KERNEL); TF_CALL_int64(REGISTER_SYCL_KERNEL); #undef REGISTER_SYCL_KERNEL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TF_CALL_float(REGISTER_CPU_KERNEL); TF_CALL_double(REGISTER_CPU_KERNEL); @@ -116,7 +116,7 @@ TF_CALL_int64(REGISTER_GPU_KERNEL); #undef REGISTER_CPU_KERNEL #undef REGISTER_GPU_KERNEL -template +template class LinSpaceOp : public OpKernel { public: explicit LinSpaceOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -136,7 +136,7 @@ class LinSpaceOp : public OpKernel { num_in.shape().DebugString())); const T start = start_in.scalar()(); const T stop = stop_in.scalar()(); - const int32 num = num_in.scalar()(); + const Tnum num = num_in.scalar()(); OP_REQUIRES(context, num > 0, errors::InvalidArgument("Requires num > 0: ", num)); Tensor* out = nullptr; @@ -147,34 +147,46 @@ class LinSpaceOp : public OpKernel { flat(0) = start; } else { const T step = (stop - start) / (num - 1); - for (int32 i = 0; i < num; ++i) flat(i) = start + step * i; + for (Tnum i = 0; i < num; ++i) flat(i) = start + step * i; } } }; -#define REGISTER_KERNEL(DEV, T) \ - REGISTER_KERNEL_BUILDER(Name("LinSpace") \ - .Device(DEV) \ - .TypeConstraint("T") \ - .TypeConstraint("Tidx") \ - .HostMemory("start") \ - .HostMemory("stop") \ - .HostMemory("num") \ - .HostMemory("output"), \ - LinSpaceOp); -#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL(DEVICE_CPU, T) +#define REGISTER_KERNEL(DEV, T, Tidx) \ + REGISTER_KERNEL_BUILDER(Name("LinSpace") \ + .Device(DEV) \ + .TypeConstraint("T") \ + .TypeConstraint("Tidx") \ + .HostMemory("start") \ + .HostMemory("stop") \ + .HostMemory("num") \ + .HostMemory("output"), \ + LinSpaceOp); + +#define REGISTER_KERNEL_ALL_NUMS(dev, T) \ + REGISTER_KERNEL(dev, T, int32); \ + REGISTER_KERNEL(dev, T, int64) + +#define REGISTER_CPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_CPU, T) TF_CALL_float(REGISTER_CPU_KERNEL); TF_CALL_double(REGISTER_CPU_KERNEL); // NOTE(touts): We register the op on GPU but it still runs on CPU // because its inputs and outputs are tagged as HostMemory. -#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL(DEVICE_GPU, T) +#define REGISTER_GPU_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_GPU, T) TF_CALL_float(REGISTER_GPU_KERNEL); TF_CALL_double(REGISTER_GPU_KERNEL); +#undef REGISTER_GPU_KERNEL #ifdef TENSORFLOW_USE_SYCL -#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL(DEVICE_SYCL, T) +#define REGISTER_SYCL_KERNEL(T) REGISTER_KERNEL_ALL_NUMS(DEVICE_SYCL, T) TF_CALL_float(REGISTER_SYCL_KERNEL); TF_CALL_double(REGISTER_SYCL_KERNEL); -#endif // TENSORFLOW_USE_SYCL +#undef REGISTER_SYCL_KERNEL +#endif // TENSORFLOW_USE_SYCL + +#undef REGISTER_CPU_KERNEL +#undef REGISTER_KERNEL_ALL_NUMS +#undef REGISTER_KERNEL + } // namespace tensorflow diff --git a/tensorflow/core/kernels/sequence_ops_test.cc b/tensorflow/core/kernels/sequence_ops_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..5f0e0a69a890aafa56b43cc55e99f490c100faa7 --- /dev/null +++ b/tensorflow/core/kernels/sequence_ops_test.cc @@ -0,0 +1,148 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/ops_testutil.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace { + +class RangeOpTest : public OpsTestBase { + protected: + void MakeOp(DataType input_type) { + TF_ASSERT_OK(NodeDefBuilder("myop", "Range") + .Input(FakeInput(input_type)) + .Input(FakeInput(input_type)) + .Input(FakeInput(input_type)) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + } +}; + +class LinSpaceOpTest : public OpsTestBase { + protected: + void MakeOp(DataType input_type, DataType index_type) { + TF_ASSERT_OK(NodeDefBuilder("myop", "LinSpace") + .Input(FakeInput(input_type)) + .Input(FakeInput(input_type)) + .Input(FakeInput(index_type)) + .Finalize(node_def())); + TF_ASSERT_OK(InitOp()); + } +}; + +TEST_F(RangeOpTest, Simple_D32) { + MakeOp(DT_INT32); + + // Feed and run + AddInputFromArray(TensorShape({}), {0}); + AddInputFromArray(TensorShape({}), {10}); + AddInputFromArray(TensorShape({}), {2}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_INT32, TensorShape({5})); + test::FillValues(&expected, {0, 2, 4, 6, 8}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(RangeOpTest, Simple_Float) { + MakeOp(DT_FLOAT); + + // Feed and run + AddInputFromArray(TensorShape({}), {0.5}); + AddInputFromArray(TensorShape({}), {2}); + AddInputFromArray(TensorShape({}), {0.3}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_FLOAT, TensorShape({5})); + test::FillValues(&expected, {0.5, 0.8, 1.1, 1.4, 1.7}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(RangeOpTest, Large_Double) { + MakeOp(DT_DOUBLE); + + // Feed and run + AddInputFromArray(TensorShape({}), {0.0}); + AddInputFromArray(TensorShape({}), {10000}); + AddInputFromArray(TensorShape({}), {0.5}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_DOUBLE, TensorShape({20000})); + std::vector result; + for (int32 i = 0; i < 20000; ++i) result.push_back(i * 0.5); + test::FillValues(&expected, gtl::ArraySlice(result)); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(LinSpaceOpTest, Simple_D32) { + MakeOp(DT_FLOAT, DT_INT32); + + // Feed and run + AddInputFromArray(TensorShape({}), {3.0}); + AddInputFromArray(TensorShape({}), {7.0}); + AddInputFromArray(TensorShape({}), {3}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_FLOAT, TensorShape({3})); + test::FillValues(&expected, {3.0, 5.0, 7.0}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(LinSpaceOpTest, Single_D64) { + MakeOp(DT_FLOAT, DT_INT64); + + // Feed and run + AddInputFromArray(TensorShape({}), {9.0}); + AddInputFromArray(TensorShape({}), {100.0}); + AddInputFromArray(TensorShape({}), {1}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_FLOAT, TensorShape({1})); + test::FillValues(&expected, {9.0}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +TEST_F(LinSpaceOpTest, Simple_Double) { + MakeOp(DT_DOUBLE, DT_INT32); + + // Feed and run + AddInputFromArray(TensorShape({}), {5.0}); + AddInputFromArray(TensorShape({}), {6.0}); + AddInputFromArray(TensorShape({}), {6}); + TF_ASSERT_OK(RunOpKernel()); + + // Check the output + Tensor expected(allocator(), DT_DOUBLE, TensorShape({6})); + test::FillValues(&expected, {5.0, 5.2, 5.4, 5.6, 5.8, 6.0}); + test::ExpectTensorEqual(expected, *GetOutput(0)); +} + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/kernels/spacetobatch_op.cc b/tensorflow/core/kernels/spacetobatch_op.cc index c513683918e9eb53768864e901d3b322b3d18879..95c1f5e7e8ca978fda334396538de0cf4ed5b774 100644 --- a/tensorflow/core/kernels/spacetobatch_op.cc +++ b/tensorflow/core/kernels/spacetobatch_op.cc @@ -248,40 +248,34 @@ class SpaceToBatchOp : public OpKernel { Tensor block_shape_; }; -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("block_shape") \ - .HostMemory("paddings"), \ - SpaceToBatchNDOp); \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("paddings"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("paddings"), \ + SpaceToBatchNDOp); \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("paddings"), \ SpaceToBatchOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER); #undef REGISTER #if GOOGLE_CUDA -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tblock_shape") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("block_shape") \ - .HostMemory("paddings"), \ - SpaceToBatchNDOp); \ - REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tpaddings") \ - .HostMemory("paddings"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatchND") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("block_shape") \ + .HostMemory("paddings"), \ + SpaceToBatchNDOp); \ + REGISTER_KERNEL_BUILDER(Name("SpaceToBatch") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("paddings"), \ SpaceToBatchOp); TF_CALL_GPU_NUMBER_TYPES(REGISTER); diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc index 94c7a0a3f695d2903106dfab3ed0c076fa962164..a1a01e8813261592a0d9ea97d6f76a163d070ee4 100644 --- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc +++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc @@ -66,10 +66,6 @@ __global__ void S2D_NCHW(const int32 nthreads, const int block_size, const int output_width, const int input_depth_by_output_height, dtype* __restrict__ output_ptr) { - // TODO(pauldonnelly): This kernel gets input coalescing, but not output - // coalescing. We could use shared memory to get both. It may also help - // to amortize the address calculations via an inner loop over block_size. - // A template parameter for the block_size is another potential optimization. CUDA_1D_KERNEL_LOOP(input_idx, nthreads) { // We assume both the input and output are packed NCHW tensors. // input_idx represents an index within the flattened input tensor. @@ -100,6 +96,48 @@ __global__ void S2D_NCHW(const int32 nthreads, } } +// Space2Depth kernel for FORMAT_NCHW using a loop over block area. +// See 'spacetodepth_op.h' for functional specification. +template +__global__ void S2D_NCHW_LOOP(const int32 nthreads, + const dtype* __restrict__ input, + const int output_width, const int input_width, + const int input_depth_by_output_area, + const int output_depth_by_output_area, + dtype* __restrict__ output) { + CUDA_1D_KERNEL_LOOP(thread_idx, nthreads) { + // We will be converting the image from ordering: + // n, iC, oY, bY, oX, bX (== input index) to + // n, bY, bX, iC, oY, oX (== output index) + + // We assume thread_idx encodes n_iC_oY_oX, and use an unrolled loop over + // bY and bX coordinates within the block. This kernel gets a small + // performance improvement compared with S2D_NCHW due to a denser access + // pattern on the input side. (Note: the equivalent D2S kernel gets a larger + // improvement as a denser pattern on the output side makes more + // difference). + + const int n_iC_oY = thread_idx / output_width; + const int oX = thread_idx - n_iC_oY * output_width; + const int n = thread_idx / input_depth_by_output_area; + const int iC_oY_oX = thread_idx - n * input_depth_by_output_area; + + // Recombine the components and apply to the input and output pointers. + auto input_ptr = input + (n_iC_oY * input_width + oX) * block_size; + auto output_ptr = output + n * output_depth_by_output_area + iC_oY_oX; + +#pragma unroll + // Copy a patch of data to the output batch image. + for (int bY = 0; bY < block_size; ++bY) { +#pragma unroll + for (int bX = 0; bX < block_size; ++bX) { + output_ptr[(bY * block_size + bX) * input_depth_by_output_area] = + ldg(input_ptr + bY * input_width + bX); + } + } + } +} + // Specialization of SpaceToDepthOpFunctor for a CPUDevice. namespace functor { template @@ -137,9 +175,40 @@ struct SpaceToDepthOpFunctor { const int output_depth = output.dimension(1); const int output_height = output.dimension(2); const int output_width = output.dimension(3); - - const int total_count = - batch_size * output_height * output_width * output_depth; + const int output_area = output_width * output_height; + const int output_depth_by_output_area = output_depth * output_area; + + // We improve performance by generating instantiations of the loop kernel + // for the most common block sizes. + if (block_size <= 4) { + const int input_width = input.dimension(3); + const int input_depth_by_output_area = input_depth * output_area; + const int total_count = batch_size * input_depth_by_output_area; + CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d); + switch (block_size) { + case 2: + return S2D_NCHW_LOOP + <<>>( + total_count, input.data(), output_width, input_width, + input_depth_by_output_area, output_depth_by_output_area, + output.data()); + case 3: + return S2D_NCHW_LOOP + <<>>( + total_count, input.data(), output_width, input_width, + input_depth_by_output_area, output_depth_by_output_area, + output.data()); + case 4: + return S2D_NCHW_LOOP + <<>>( + total_count, input.data(), output_width, input_width, + input_depth_by_output_area, output_depth_by_output_area, + output.data()); + } + } + + // Other block sizes are processed by the generic kernel. + const int total_count = batch_size * output_depth_by_output_area; CudaLaunchConfig config = GetCudaLaunchConfig(total_count, d); S2D_NCHW<<>>( config.virtual_thread_count, input.data(), block_size, output_width, diff --git a/tensorflow/core/kernels/split_op.cc b/tensorflow/core/kernels/split_op.cc index 4d2100c59ca27199955fe3430aa004984efd764a..58e1a73be61cf04aba05ebadb8d8e49f6aacef6b 100644 --- a/tensorflow/core/kernels/split_op.cc +++ b/tensorflow/core/kernels/split_op.cc @@ -167,11 +167,11 @@ class SplitOpCPU : public SplitOpBase { const auto num_threads = context->device()->tensorflow_cpu_worker_threads()->num_threads; // TODO(jewillco): Tune heuristic further. + const auto input_element_count = input_shape.num_elements(); const bool use_parallelism_between_outputs = (num_split >= 4 && - input_shape.num_elements() >= - std::max(num_threads, num_split) * 4096 && - input_shape.num_elements() < num_split * 180 * 1024); + input_element_count >= std::max(num_threads, num_split) * 4096 && + input_element_count < num_split * 180 * 1024); auto range_output_func = [&indices, context, &output_shape, prefix_dim_size, split_dim_output_size, suffix_dim_size, &sizes, @@ -209,7 +209,7 @@ class SplitOpCPU : public SplitOpBase { // Run in parallel, disabling parallelism in functor. Shard(num_split, context->device()->tensorflow_cpu_worker_threads()->workers, - num_split, kint64max, range_output_func); + num_split, input_element_count / num_split, range_output_func); } else { // Run sequentially, but allow internal parallelism in functor. range_output_func(0, num_split); diff --git a/tensorflow/core/kernels/split_v_op.cc b/tensorflow/core/kernels/split_v_op.cc index e2dd66da1eb5063f53b2c07106196473918832bb..3316e5fcc920166a8bd4f49f4ce1752b4c8910cb 100644 --- a/tensorflow/core/kernels/split_v_op.cc +++ b/tensorflow/core/kernels/split_v_op.cc @@ -225,11 +225,11 @@ class SplitVOpCPU : public SplitVOpBase { const auto num_threads = context->device()->tensorflow_cpu_worker_threads()->num_threads; // TODO(jewillco): Tune heuristic further. + const auto input_element_count = input_shape.num_elements(); const bool use_parallelism_between_outputs = (num_split >= 4 && - input_shape.num_elements() >= - std::max(num_threads, num_split) * 4096 && - input_shape.num_elements() < num_split * 180 * 1024); + input_element_count >= std::max(num_threads, num_split) * 4096 && + input_element_count < num_split * 180 * 1024); auto range_output_func = [&indices, context, &input_shape, prefix_dim_size, split_dim, &split_sizes_vec, &split_start_points, @@ -267,7 +267,7 @@ class SplitVOpCPU : public SplitVOpBase { // Run in parallel, disabling parallelism in functor. Shard(num_split, context->device()->tensorflow_cpu_worker_threads()->workers, - num_split, kint64max, range_output_func); + num_split, input_element_count / num_split, range_output_func); } else { // Run sequentially, but allow internal parallelism in functor. range_output_func(0, num_split); diff --git a/tensorflow/core/kernels/sql/sqlite_query_connection.cc b/tensorflow/core/kernels/sql/sqlite_query_connection.cc index a9e6ee09694768049cfced0426c208de520bae0f..1330506d28ca96b4a9e668219dc67cbb1c3b796d 100644 --- a/tensorflow/core/kernels/sql/sqlite_query_connection.cc +++ b/tensorflow/core/kernels/sql/sqlite_query_connection.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ #include "tensorflow/core/kernels/sql/sqlite_query_connection.h" + #include "tensorflow/core/lib/strings/stringprintf.h" namespace tensorflow { @@ -29,17 +30,18 @@ Status SqliteQueryConnection::Open(const string& data_source_name, return errors::FailedPrecondition( "Failed to open query connection: Connection already opeend."); } - Status s = db::Sqlite::Open(data_source_name, &db_); + auto s = Sqlite::Open(data_source_name); if (s.ok()) { + db_ = std::move(s.ValueOrDie()); query_ = query; output_types_ = output_types; } - return s; + return s.status(); } Status SqliteQueryConnection::Close() { Status s; - s.Update(stmt_->Close()); + s.Update(stmt_.Close()); s.Update(db_->Close()); return s; } @@ -52,7 +54,7 @@ Status SqliteQueryConnection::GetNext(std::vector* out_tensors, return s; } } - Status s = stmt_->Step(end_of_sequence); + Status s = stmt_.Step(end_of_sequence); if (!*end_of_sequence) { for (int i = 0; i < column_count_; i++) { DataType dt = output_types_[i]; @@ -66,9 +68,9 @@ Status SqliteQueryConnection::GetNext(std::vector* out_tensors, Status SqliteQueryConnection::PrepareQuery() { stmt_ = db_->Prepare(query_); - Status s = stmt_->status(); + Status s = stmt_.status(); if (s.ok()) { - int column_count = stmt_->ColumnCount(); + int column_count = stmt_.ColumnCount(); if (column_count != output_types_.size()) { return errors::InvalidArgument(tensorflow::strings::Printf( "The number of columns in query (%d) must match the number of " @@ -84,40 +86,40 @@ void SqliteQueryConnection::FillTensorWithResultSetEntry( const DataType& data_type, int column_index, Tensor* tensor) { switch (data_type) { case DT_STRING: - tensor->scalar()() = stmt_->ColumnString(column_index); + tensor->scalar()() = stmt_.ColumnString(column_index); break; case DT_INT8: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_INT16: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_INT32: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_INT64: - tensor->scalar()() = stmt_->ColumnInt(column_index); + tensor->scalar()() = stmt_.ColumnInt(column_index); break; case DT_UINT8: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_UINT16: tensor->scalar()() = - static_cast(stmt_->ColumnInt(column_index)); + static_cast(stmt_.ColumnInt(column_index)); break; case DT_BOOL: - tensor->scalar()() = stmt_->ColumnInt(column_index) != 0; + tensor->scalar()() = stmt_.ColumnInt(column_index) != 0; break; case DT_FLOAT: tensor->scalar()() = - static_cast(stmt_->ColumnDouble(column_index)); + static_cast(stmt_.ColumnDouble(column_index)); break; case DT_DOUBLE: - tensor->scalar()() = stmt_->ColumnDouble(column_index); + tensor->scalar()() = stmt_.ColumnDouble(column_index); break; // Error preemptively thrown by SqlDatasetOp::MakeDataset in this case. default: { diff --git a/tensorflow/core/kernels/sql/sqlite_query_connection.h b/tensorflow/core/kernels/sql/sqlite_query_connection.h index b0b4737a1ea2ff3045c1ce4b2400ebe829e49f91..435dd8e234ca7a8fb9a3ef6ffeef0ca4dda7a221 100644 --- a/tensorflow/core/kernels/sql/sqlite_query_connection.h +++ b/tensorflow/core/kernels/sql/sqlite_query_connection.h @@ -42,8 +42,8 @@ class SqliteQueryConnection : public QueryConnection { // `stmt_`. void FillTensorWithResultSetEntry(const DataType& data_type, int column_index, Tensor* tensor); - std::unique_ptr db_ = nullptr; - std::unique_ptr stmt_ = nullptr; + std::shared_ptr db_ = nullptr; + SqliteStatement stmt_; int column_count_ = 0; string query_; DataTypeVector output_types_; diff --git a/tensorflow/core/kernels/stateless_random_ops.cc b/tensorflow/core/kernels/stateless_random_ops.cc index 79d0c07acdee4f3e9e979c00bf8ccbf2853a77cc..f6fb0a121d8336a1abd624103e33e3ed8869f0d2 100644 --- a/tensorflow/core/kernels/stateless_random_ops.cc +++ b/tensorflow/core/kernels/stateless_random_ops.cc @@ -137,7 +137,6 @@ TF_CALL_double(REGISTER); .Device(DEVICE_GPU) \ .HostMemory("shape") \ .HostMemory("seed") \ - .TypeConstraint("T") \ .TypeConstraint("dtype"), \ StatelessRandomOp >); \ @@ -146,7 +145,6 @@ TF_CALL_double(REGISTER); .Device(DEVICE_GPU) \ .HostMemory("shape") \ .HostMemory("seed") \ - .TypeConstraint("T") \ .TypeConstraint("dtype"), \ StatelessRandomOp >); \ @@ -155,7 +153,6 @@ TF_CALL_double(REGISTER); .Device(DEVICE_GPU) \ .HostMemory("shape") \ .HostMemory("seed") \ - .TypeConstraint("T") \ .TypeConstraint("dtype"), \ StatelessRandomOp< \ GPUDevice, \ diff --git a/tensorflow/core/kernels/summary_interface.cc b/tensorflow/core/kernels/summary_interface.cc index e95a4c7b89f8b20dd1374f493bb4eb0bbf46963b..313137ae4957a086be57b490fe1a5f6f95e93f0f 100644 --- a/tensorflow/core/kernels/summary_interface.cc +++ b/tensorflow/core/kernels/summary_interface.cc @@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/core/kernels/summary_interface.h" + +#include #include "tensorflow/compiler/xla/ptr_util.h" #include "tensorflow/core/framework/op_kernel.h" @@ -19,18 +22,16 @@ limitations under the License. #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/kernels/summary_interface.h" #include "tensorflow/core/lib/histogram/histogram.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/lib/png/png_io.h" #include "tensorflow/core/lib/wav/wav_io.h" -#include "tensorflow/core/util/event.pb.h" #include "tensorflow/core/util/events_writer.h" namespace tensorflow { namespace { template -Status TensorValueAt(Tensor t, int index, T* out) { +Status TensorValueAt(Tensor t, int64 index, T* out) { switch (t.dtype()) { case DT_FLOAT: *out = t.flat()(index); @@ -210,20 +211,20 @@ Status NormalizeAndAddImages(const Tensor& tensor, int max_images, int h, int w, class SummaryWriterImpl : public SummaryWriterInterface { public: - SummaryWriterImpl(int max_queue, int flush_millis) + SummaryWriterImpl(int max_queue, int flush_millis, Env* env) : SummaryWriterInterface(), is_initialized_(false), max_queue_(max_queue), - flush_millis_(flush_millis) {} + flush_millis_(flush_millis), + env_(env) {} - Status Initialize(const string& logdir, const string& filename_suffix, - Env* env) { - const Status is_dir = env->IsDirectory(logdir); + Status Initialize(const string& logdir, const string& filename_suffix) { + const Status is_dir = env_->IsDirectory(logdir); if (!is_dir.ok()) { if (is_dir.code() != tensorflow::error::NOT_FOUND) { return is_dir; } - TF_RETURN_IF_ERROR(env->CreateDir(logdir)); + TF_RETURN_IF_ERROR(env_->CreateDir(logdir)); } mutex_lock ml(mu_); events_writer_ = @@ -231,7 +232,7 @@ class SummaryWriterImpl : public SummaryWriterInterface { if (!events_writer_->InitWithSuffix(filename_suffix)) { return errors::Unknown("Could not initialize events writer."); } - last_flush_ = Env::Default()->NowMicros(); + last_flush_ = env_->NowMicros(); is_initialized_ = true; return Status::OK(); } @@ -250,28 +251,34 @@ class SummaryWriterImpl : public SummaryWriterInterface { Status WriteTensor(int64 global_step, Tensor t, const string& tag, const string& serialized_metadata) override { - Summary s; - Summary::Value* v = s.add_value(); + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary::Value* v = e->mutable_summary()->add_value(); t.AsProtoTensorContent(v->mutable_tensor()); v->set_tag(tag); v->mutable_metadata()->ParseFromString(serialized_metadata); - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteScalar(int64 global_step, Tensor t, const string& tag) override { - Summary s; - Summary::Value* v = s.add_value(); + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary::Value* v = e->mutable_summary()->add_value(); v->set_tag(tag); float value; TF_RETURN_IF_ERROR(TensorValueAt(t, 0, &value)); v->set_simple_value(value); - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteHistogram(int64 global_step, Tensor t, const string& tag) override { - Summary s; - Summary::Value* v = s.add_value(); + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary::Value* v = e->mutable_summary()->add_value(); v->set_tag(tag); histogram::Histogram histo; for (int64 i = 0; i < t.NumElements(); i++) { @@ -287,7 +294,7 @@ class SummaryWriterImpl : public SummaryWriterInterface { } histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteImage(int64 global_step, Tensor tensor, const string& tag, @@ -306,7 +313,10 @@ class SummaryWriterImpl : public SummaryWriterInterface { return errors::InvalidArgument("Tensor too large for summary ", tensor.shape().DebugString()); } - Summary s; + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary* s = e->mutable_summary(); // The casts and h * w cannot overflow because of the limits above. const int batch_size = static_cast(tensor.dim_size(0)); const int h = static_cast(tensor.dim_size(1)); @@ -321,20 +331,20 @@ class SummaryWriterImpl : public SummaryWriterInterface { &values(i, 0, 0), Eigen::DSizes(hw, depth)); }; TF_RETURN_IF_ERROR( - AddImages(tag, max_images, batch_size, w, h, depth, ith_image, &s)); + AddImages(tag, max_images, batch_size, w, h, depth, ith_image, s)); } else if (tensor.dtype() == DT_HALF) { TF_RETURN_IF_ERROR(NormalizeAndAddImages( - tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s)); + tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s)); } else if (tensor.dtype() == DT_FLOAT) { TF_RETURN_IF_ERROR(NormalizeAndAddImages( - tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, &s)); + tensor, max_images, h, w, hw, depth, batch_size, tag, bad_color, s)); } else { return errors::InvalidArgument( "Only DT_INT8, DT_HALF, and DT_FLOAT images are supported. Got ", DataTypeString(tensor.dtype())); } - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } Status WriteAudio(int64 global_step, Tensor tensor, const string& tag, @@ -346,10 +356,13 @@ class SummaryWriterImpl : public SummaryWriterInterface { const int64 length_frames = tensor.dim_size(1); const int64 num_channels = tensor.dims() == 2 ? 1 : tensor.dim_size(tensor.dims() - 1); - Summary s; + std::unique_ptr e{new Event}; + e->set_step(global_step); + e->set_wall_time(GetWallTime()); + Summary* s = e->mutable_summary(); const int N = std::min(max_outputs, batch_size); for (int i = 0; i < N; ++i) { - Summary::Value* v = s.add_value(); + Summary::Value* v = s->add_value(); if (max_outputs > 1) { v->set_tag(strings::StrCat(tag, "/audio/", i)); } else { @@ -375,36 +388,35 @@ class SummaryWriterImpl : public SummaryWriterInterface { channels_by_frames.data(), sample_rate_truncated, num_channels, length_frames, sa->mutable_encoded_audio_string())); } - - return Enqueue(global_step, s); + return WriteEvent(std::move(e)); } - string DebugString() override { return "SummaryWriterImpl"; } - - private: - Status Enqueue(int64 global_step, const Summary& summary) { + Status WriteEvent(std::unique_ptr event) override { mutex_lock ml(mu_); - queue_.emplace_back(global_step, summary, Env::Default()->NowMicros()); + queue_.emplace_back(std::move(event)); if (queue_.size() >= max_queue_ || - Env::Default()->NowMicros() - last_flush_ > 1000 * flush_millis_) { + env_->NowMicros() - last_flush_ > 1000 * flush_millis_) { return InternalFlush(); } return Status::OK(); } + string DebugString() override { return "SummaryWriterImpl"; } + + private: + double GetWallTime() { + return static_cast(env_->NowMicros()) / 1.0e6; + } + Status InternalFlush() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - for (const EventInfo& e : queue_) { - Event event; - event.set_step(std::get<0>(e)); - *event.mutable_summary() = std::get<1>(e); - event.set_wall_time(std::get<2>(e)); - events_writer_->WriteEvent(event); + for (const std::unique_ptr& e : queue_) { + events_writer_->WriteEvent(*e); } queue_.clear(); if (!events_writer_->Flush()) { return errors::InvalidArgument("Could not flush events file."); } - last_flush_ = Env::Default()->NowMicros(); + last_flush_ = env_->NowMicros(); return Status::OK(); } @@ -412,9 +424,9 @@ class SummaryWriterImpl : public SummaryWriterInterface { const int max_queue_; const int flush_millis_; uint64 last_flush_; - using EventInfo = std::tuple; + Env* env_; mutex mu_; - std::vector queue_ GUARDED_BY(mu_); + std::vector> queue_ GUARDED_BY(mu_); // A pointer to allow deferred construction. std::unique_ptr events_writer_ GUARDED_BY(mu_); std::vector> registered_summaries_ @@ -424,8 +436,8 @@ class SummaryWriterImpl : public SummaryWriterInterface { Status CreateSummaryWriter(int max_queue, int flush_millis, const string& logdir, const string& filename_suffix, Env* env, SummaryWriterInterface** result) { - SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis); - const Status s = w->Initialize(logdir, filename_suffix, env); + SummaryWriterImpl* w = new SummaryWriterImpl(max_queue, flush_millis, env); + const Status s = w->Initialize(logdir, filename_suffix); if (!s.ok()) { w->Unref(); *result = nullptr; diff --git a/tensorflow/core/kernels/summary_interface.h b/tensorflow/core/kernels/summary_interface.h index ae2fbb70fe3580bdd1d4f4f34a487b33f5a6a9c2..ccf3459e56b690522f9551d9c1fed4e649455814 100644 --- a/tensorflow/core/kernels/summary_interface.h +++ b/tensorflow/core/kernels/summary_interface.h @@ -15,8 +15,10 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ #define TENSORFLOW_CORE_KERNELS_SUMMARY_INTERFACE_H_ +#include #include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/util/event.pb.h" namespace tensorflow { @@ -43,13 +45,16 @@ class SummaryWriterInterface : public ResourceBase { virtual Status WriteAudio(int64 global_step, Tensor t, const string& tag, int max_outputs_, float sample_rate) = 0; + + virtual Status WriteEvent(std::unique_ptr e) = 0; }; // Creates a SummaryWriterInterface instance which writes to a file. It will // enqueue up to max_queue summaries, and flush at least every flush_millis // milliseconds. The summaries will be written to the directory specified by // logdir and with the filename suffixed by filename_suffix. The caller owns a -// reference to result if the returned status is ok. +// reference to result if the returned status is ok. The Env object must not +// be destroyed until after the returned writer. Status CreateSummaryWriter(int max_queue, int flush_millis, const string& logdir, const string& filename_suffix, Env* env, SummaryWriterInterface** result); diff --git a/tensorflow/core/kernels/summary_interface_test.cc b/tensorflow/core/kernels/summary_interface_test.cc index 0e24e8122a0760980ffed69790c482175f4623e3..58e021a0b3e889ce1efe1bb5c73bcc74e16db139 100644 --- a/tensorflow/core/kernels/summary_interface_test.cc +++ b/tensorflow/core/kernels/summary_interface_test.cc @@ -12,11 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ - -#include +#include "tensorflow/core/kernels/summary_interface.h" #include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/kernels/summary_interface.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/refcount.h" #include "tensorflow/core/lib/io/path.h" @@ -28,52 +26,68 @@ limitations under the License. namespace tensorflow { namespace { -Status SummaryTestHelper( - const string& test_name, - std::function writer_fn, - std::function test_fn) { - static std::set* tests = new std::set(); - CHECK(tests->insert(test_name).second) << ": " << test_name; - - SummaryWriterInterface* writer; - Env* env = Env::Default(); - TF_CHECK_OK( - CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, env, &writer)); - core::ScopedUnref deleter(writer); - - TF_CHECK_OK(writer_fn(writer)); - TF_CHECK_OK(writer->Flush()); - - std::vector files; - TF_CHECK_OK(env->GetChildren(testing::TmpDir(), &files)); - bool found = false; - for (const string& f : files) { - if (StringPiece(f).contains(test_name)) { - if (found) { - return errors::Unknown("Found more than one file for ", test_name); +class FakeClockEnv : public EnvWrapper { + public: + FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {} + void AdvanceByMillis(const uint64 millis) { current_millis_ += millis; } + uint64 NowMicros() override { return current_millis_ * 1000; } + uint64 NowSeconds() override { return current_millis_ * 1000; } + + private: + uint64 current_millis_; +}; + +class SummaryInterfaceTest : public ::testing::Test { + protected: + Status SummaryTestHelper( + const string& test_name, + const std::function& writer_fn, + const std::function& test_fn) { + static std::set* tests = new std::set(); + CHECK(tests->insert(test_name).second) << ": " << test_name; + + SummaryWriterInterface* writer; + TF_CHECK_OK(CreateSummaryWriter(1, 1, testing::TmpDir(), test_name, &env_, + &writer)); + core::ScopedUnref deleter(writer); + + TF_CHECK_OK(writer_fn(writer)); + TF_CHECK_OK(writer->Flush()); + + std::vector files; + TF_CHECK_OK(env_.GetChildren(testing::TmpDir(), &files)); + bool found = false; + for (const string& f : files) { + if (StringPiece(f).contains(test_name)) { + if (found) { + return errors::Unknown("Found more than one file for ", test_name); + } + found = true; + std::unique_ptr read_file; + TF_CHECK_OK(env_.NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f), + &read_file)); + io::RecordReader reader(read_file.get(), io::RecordReaderOptions()); + string record; + uint64 offset = 0; + TF_CHECK_OK( + reader.ReadRecord(&offset, + &record)); // The first event is irrelevant + TF_CHECK_OK(reader.ReadRecord(&offset, &record)); + Event e; + e.ParseFromString(record); + test_fn(e); } - found = true; - std::unique_ptr read_file; - TF_CHECK_OK(env->NewRandomAccessFile(io::JoinPath(testing::TmpDir(), f), - &read_file)); - io::RecordReader reader(read_file.get(), io::RecordReaderOptions()); - string record; - uint64 offset = 0; - TF_CHECK_OK(reader.ReadRecord(&offset, - &record)); // The first event is irrelevant - TF_CHECK_OK(reader.ReadRecord(&offset, &record)); - Event e; - e.ParseFromString(record); - test_fn(e); } + if (!found) { + return errors::Unknown("Found no file for ", test_name); + } + return Status::OK(); } - if (!found) { - return errors::Unknown("Found no file for ", test_name); - } - return Status::OK(); -} -TEST(SummaryInterfaceTest, WriteTensor) { + FakeClockEnv env_; +}; + +TEST_F(SummaryInterfaceTest, WriteTensor) { TF_CHECK_OK(SummaryTestHelper("tensor_test", [](SummaryWriterInterface* writer) { Tensor one(DT_FLOAT, TensorShape({})); @@ -91,7 +105,7 @@ TEST(SummaryInterfaceTest, WriteTensor) { })); } -TEST(SummaryInterfaceTest, WriteScalar) { +TEST_F(SummaryInterfaceTest, WriteScalar) { TF_CHECK_OK(SummaryTestHelper( "scalar_test", [](SummaryWriterInterface* writer) { @@ -109,7 +123,7 @@ TEST(SummaryInterfaceTest, WriteScalar) { })); } -TEST(SummaryInterfaceTest, WriteHistogram) { +TEST_F(SummaryInterfaceTest, WriteHistogram) { TF_CHECK_OK(SummaryTestHelper("hist_test", [](SummaryWriterInterface* writer) { Tensor one(DT_FLOAT, TensorShape({})); @@ -127,7 +141,7 @@ TEST(SummaryInterfaceTest, WriteHistogram) { })); } -TEST(SummaryInterfaceTest, WriteImage) { +TEST_F(SummaryInterfaceTest, WriteImage) { TF_CHECK_OK(SummaryTestHelper( "image_test", [](SummaryWriterInterface* writer) { @@ -148,7 +162,7 @@ TEST(SummaryInterfaceTest, WriteImage) { })); } -TEST(SummaryInterfaceTest, WriteAudio) { +TEST_F(SummaryInterfaceTest, WriteAudio) { TF_CHECK_OK(SummaryTestHelper( "audio_test", [](SummaryWriterInterface* writer) { @@ -166,5 +180,37 @@ TEST(SummaryInterfaceTest, WriteAudio) { })); } +TEST_F(SummaryInterfaceTest, WriteEvent) { + TF_CHECK_OK( + SummaryTestHelper("event_test", + [](SummaryWriterInterface* writer) { + std::unique_ptr e{new Event}; + e->set_step(7); + e->mutable_summary()->add_value()->set_tag("hi"); + TF_RETURN_IF_ERROR(writer->WriteEvent(std::move(e))); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { + EXPECT_EQ(e.step(), 7); + CHECK_EQ(e.summary().value_size(), 1); + EXPECT_EQ(e.summary().value(0).tag(), "hi"); + })); +} + +TEST_F(SummaryInterfaceTest, WallTime) { + env_.AdvanceByMillis(7023); + TF_CHECK_OK(SummaryTestHelper( + "wall_time_test", + [](SummaryWriterInterface* writer) { + Tensor one(DT_FLOAT, TensorShape({})); + one.scalar()() = 1.0; + TF_RETURN_IF_ERROR(writer->WriteScalar(2, one, "name")); + TF_RETURN_IF_ERROR(writer->Flush()); + return Status::OK(); + }, + [](const Event& e) { EXPECT_EQ(e.wall_time(), 7.023); })); +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/kernels/svd_op_gpu.cu.cc b/tensorflow/core/kernels/svd_op_gpu.cu.cc index 1603a8aeda97a50161d86fd11fc2f76c5c70b398..dedc2da60bab0d0c0613630c384c2f23ddae31e3 100644 --- a/tensorflow/core/kernels/svd_op_gpu.cu.cc +++ b/tensorflow/core/kernels/svd_op_gpu.cu.cc @@ -190,8 +190,8 @@ class SvdOpGpu : public AsyncOpKernel { // TODO: can the two cases (MgeqN and MlessN) be simplified, // common boilerplate be reduced, or even combined in one method? void PerformSVD_MgeqN(OpKernelContext* context, DoneCallback done, int64 m, - int64 n, int64 p, const gtl::ArraySlice& perm, - const Tensor& M, Tensor* S, Tensor* U, Tensor* V) { + int64 n, int64 p, const Tensor& M, Tensor* S, Tensor* U, + Tensor* V) { TensorShape shapeRaw = M.shape(); shapeRaw.RemoveLastDims(2); @@ -207,7 +207,7 @@ class SvdOpGpu : public AsyncOpKernel { solver->allocate_scoped_tensor(M.dtype(), input_shape, &input_copy), done); auto device = context->eigen_device(); - OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, M, perm, &input_copy), + OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, M, &input_copy), done); // I need to transpose U at the end @@ -250,7 +250,7 @@ class SvdOpGpu : public AsyncOpKernel { // Transpose U if (compute_uv_) { - OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, u_copy, perm, U), done); + OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, u_copy, U), done); } // now check if the SVD operation succeeded or not @@ -259,8 +259,8 @@ class SvdOpGpu : public AsyncOpKernel { // The SVD if m < n void PerformSVD_MlessN(OpKernelContext* context, DoneCallback done, int64 m, - int64 n, int64 p, const gtl::ArraySlice& perm, - const Tensor& M, Tensor* S, Tensor* U, Tensor* V) { + int64 n, int64 p, const Tensor& M, Tensor* S, + Tensor* U, Tensor* V) { // Perform the SVD on M' // Reuse the input buffer or make a copy for the SVD depending on whether @@ -325,7 +325,7 @@ class SvdOpGpu : public AsyncOpKernel { // Transpose V if (compute_uv_) { auto device = context->eigen_device(); - OP_REQUIRES_OK_ASYNC(context, DoTranspose(device, v_copy, perm, V), done); + OP_REQUIRES_OK_ASYNC(context, DoMatrixTranspose(device, v_copy, V), done); } // now check if the SVD operation succeeded or not @@ -389,19 +389,12 @@ class SvdOpGpu : public AsyncOpKernel { return; } - // Prepare permutation - std::vector perm; - for (size_t i = 0; i < ndims - 2; ++i) perm.push_back(i); - perm.push_back(ndims - 1); // transpose last two dimensions - perm.push_back(ndims - 2); - gtl::ArraySlice permAS(perm); - // call implementations if (m >= n) { - PerformSVD_MgeqN(context, done, m, n, p, permAS, input, outputS, outputU, + PerformSVD_MgeqN(context, done, m, n, p, input, outputS, outputU, outputV); } else { - PerformSVD_MlessN(context, done, m, n, p, permAS, input, outputS, outputU, + PerformSVD_MlessN(context, done, m, n, p, input, outputS, outputU, outputV); } } diff --git a/tensorflow/core/kernels/tensor_dataset_op.cc b/tensorflow/core/kernels/tensor_dataset_op.cc index 36caf965d7d2db998eb00aa09eef629bb9f0e5c9..db7c94732873d88c343e52036a91c3da0f549f81 100644 --- a/tensorflow/core/kernels/tensor_dataset_op.cc +++ b/tensorflow/core/kernels/tensor_dataset_op.cc @@ -40,14 +40,14 @@ class TensorDatasetOp : public DatasetOpKernel { for (const Tensor& t : inputs) { components.push_back(t); } - *output = new Dataset(std::move(components)); + *output = new Dataset(ctx, std::move(components)); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - explicit Dataset(std::vector tensors) - : tensors_(std::move(tensors)) { + Dataset(OpKernelContext* ctx, std::vector tensors) + : GraphDatasetBase(ctx), tensors_(std::move(tensors)) { for (const Tensor& t : tensors_) { dtypes_.push_back(t.dtype()); shapes_.emplace_back(t.shape().dim_sizes()); @@ -67,6 +67,21 @@ class TensorDatasetOp : public DatasetOpKernel { string DebugString() override { return "TensorDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + std::vector components; + components.reserve(tensors_.size()); + for (const Tensor& t : tensors_) { + Node* node; + TF_RETURN_IF_ERROR(b->AddTensor(t, &node)); + components.emplace_back(node); + } + TF_RETURN_IF_ERROR( + b->AddDatasetWithInputAsList(this, components, output)); + return Status::OK(); + } + private: class Iterator : public DatasetIterator { public: @@ -88,6 +103,21 @@ class TensorDatasetOp : public DatasetOpKernel { } } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + if (produced_) + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("produced"), "")); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + produced_ = reader->Contains(full_name("produced")); + return Status::OK(); + } + private: mutex mu_; bool produced_ GUARDED_BY(mu_); diff --git a/tensorflow/core/kernels/tensor_slice_dataset_op.cc b/tensorflow/core/kernels/tensor_slice_dataset_op.cc index 7b652401bc47b3c1b15766001e7cfe0dd853ea66..fd36bf524ce2570c2af94d4daafea7d0f2ad189a 100644 --- a/tensorflow/core/kernels/tensor_slice_dataset_op.cc +++ b/tensorflow/core/kernels/tensor_slice_dataset_op.cc @@ -50,14 +50,14 @@ class TensorSliceDatasetOp : public DatasetOpKernel { errors::InvalidArgument( "All components must have the same size in the 0th dimension")); } - *output = new Dataset(std::move(components)); + *output = new Dataset(ctx, std::move(components)); } private: - class Dataset : public DatasetBase { + class Dataset : public GraphDatasetBase { public: - explicit Dataset(std::vector tensors) - : tensors_(std::move(tensors)) { + explicit Dataset(OpKernelContext* ctx, std::vector tensors) + : GraphDatasetBase(ctx), tensors_(std::move(tensors)) { for (const Tensor& t : tensors_) { dtypes_.push_back(t.dtype()); gtl::InlinedVector partial_dim_sizes; @@ -83,6 +83,21 @@ class TensorSliceDatasetOp : public DatasetOpKernel { string DebugString() override { return "TensorSliceDatasetOp::Dataset"; } + protected: + Status AsGraphDefInternal(DatasetGraphDefBuilder* b, + Node** output) const override { + std::vector components; + components.reserve(tensors_.size()); + for (const Tensor& t : tensors_) { + Node* node; + TF_RETURN_IF_ERROR(b->AddTensor(t, &node)); + components.emplace_back(node); + } + TF_RETURN_IF_ERROR( + b->AddDatasetWithInputAsList(this, components, output)); + return Status::OK(); + } + private: template static Status HandleSliceToElement(const Tensor& parent, Tensor* element, @@ -148,10 +163,24 @@ class TensorSliceDatasetOp : public DatasetOpKernel { return Status::OK(); } + protected: + Status SaveInternal(IteratorStateWriter* writer) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(writer->WriteScalar(full_name("i"), i_)); + return Status::OK(); + } + + Status RestoreInternal(OpKernelContext* ctx, + IteratorStateReader* reader) override { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(reader->ReadScalar(full_name("i"), &i_)); + return Status::OK(); + } + private: mutex mu_; - int i_ GUARDED_BY(mu_); - const int n_; + int64 i_ GUARDED_BY(mu_); + const int64 n_; }; const std::vector tensors_; diff --git a/tensorflow/core/kernels/tile_functor.h b/tensorflow/core/kernels/tile_functor.h index 28af2dace3a523a32ec7be78580f68965d8663cd..189be9239ba8e5717228b611e09a783cd5503b0f 100644 --- a/tensorflow/core/kernels/tile_functor.h +++ b/tensorflow/core/kernels/tile_functor.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_KERNELS_TILE_FUNCTOR_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/types.h" @@ -29,13 +30,13 @@ namespace internal { template void TileSimple(const Device& d, Tensor* out, const Tensor& in); -template +template void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice& broadcast_array) { + const gtl::ArraySlice& broadcast_array) { auto x = in.tensor(); auto y = out->tensor(); - Eigen::array b; + Eigen::array b; for (int i = 0; i < NDIM; ++i) b[i] = broadcast_array[i]; if (Eigen::internal::is_same::value) { // Use 32bit indexing to speed up the computations @@ -45,9 +46,9 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, } } -template +template void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice&) { + const gtl::ArraySlice&) { auto x = in.tensor(); auto y = out->tensor(); // In the scalar case we simply copy the input. @@ -58,34 +59,42 @@ void TileUsingEigen(const Device& d, Tensor* out, const Tensor& in, namespace functor { -template +template struct Tile { void operator()(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice broadcast_array) const { + const gtl::ArraySlice broadcast_array) const { switch (in.dims()) { case 0: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 1: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 2: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 3: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 4: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 5: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 6: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; case 7: - internal::TileUsingEigen(d, out, in, broadcast_array); + internal::TileUsingEigen(d, out, in, + broadcast_array); break; default: internal::TileSimple(d, out, in); diff --git a/tensorflow/core/kernels/tile_functor_cpu.cc b/tensorflow/core/kernels/tile_functor_cpu.cc index 5952d492215fe6b9ddc5a933252d662fcd11f342..b2fd669541d32406512c4618fac77604baefedbe 100644 --- a/tensorflow/core/kernels/tile_functor_cpu.cc +++ b/tensorflow/core/kernels/tile_functor_cpu.cc @@ -15,10 +15,10 @@ limitations under the License. #define EIGEN_USE_THREADS +#include "tensorflow/core/kernels/tile_functor.h" #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/kernels/tile_functor.h" namespace tensorflow { @@ -51,7 +51,9 @@ namespace functor { typedef Eigen::ThreadPoolDevice CPUDevice; // Register functors used for Tile functor. -#define DEFINE_TYPE(T) template struct Tile; +#define DEFINE_TYPE(T) \ + template struct Tile; \ + template struct Tile; TF_CALL_bool(DEFINE_TYPE); TF_CALL_float(DEFINE_TYPE); @@ -70,7 +72,9 @@ TF_CALL_string(DEFINE_TYPE); #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#define DEFINE_TYPE(T) template struct Tile; +#define DEFINE_TYPE(T) \ + template struct Tile; \ + template struct Tile; TF_CALL_bool(DEFINE_TYPE); TF_CALL_float(DEFINE_TYPE); @@ -81,7 +85,7 @@ TF_CALL_int16(DEFINE_TYPE); TF_CALL_int64(DEFINE_TYPE); #undef DEFINE_TYPE -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL } // end namespace functor } // end namespace tensorflow diff --git a/tensorflow/core/kernels/tile_functor_gpu.cu.cc b/tensorflow/core/kernels/tile_functor_gpu.cu.cc index 1c61c3030ae10492d5a1ba0fb1aac23ec1da84c4..5a36e7567beb16e447de28d3cf930fbd29f6c078 100644 --- a/tensorflow/core/kernels/tile_functor_gpu.cu.cc +++ b/tensorflow/core/kernels/tile_functor_gpu.cu.cc @@ -18,10 +18,11 @@ limitations under the License. #define EIGEN_USE_GPU #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/tile_functor.h" + +#include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/kernels/tile_functor.h" #include "tensorflow/core/util/cuda_kernel_helper.h" -#include "tensorflow/core/framework/register_types.h" namespace tensorflow { namespace internal { @@ -60,7 +61,8 @@ void TileSimple(const Device& d, Tensor* out, const Tensor& in) { host_buf[ndims + i] = out_strides[i]; host_buf[ndims * 2 + i] = in.dim_size(i); } - // Copies the input strides, output strides and input dimension sizes to the device. + // Copies the input strides, output strides and input dimension sizes to the + // device. auto num_bytes = sizeof(int64) * host_buf.size(); auto dev_buf = d.allocate(num_bytes); // NOTE: host_buf is not allocated by CudaHostAllocator, and @@ -84,7 +86,9 @@ namespace functor { typedef Eigen::GpuDevice GPUDevice; // Register functors used for Tile functor. -#define DEFINE_TYPE(T) template struct Tile; +#define DEFINE_TYPE(T) \ + template struct Tile; \ + template struct Tile; TF_CALL_int16(DEFINE_TYPE); TF_CALL_int32(DEFINE_TYPE); diff --git a/tensorflow/core/kernels/tile_ops.cc b/tensorflow/core/kernels/tile_ops.cc index c49ebc06852d6e4b62ede589cb42018f210d1f28..fa5afe6a31b0c660151070f5cd2e1d5be280adc5 100644 --- a/tensorflow/core/kernels/tile_ops.cc +++ b/tensorflow/core/kernels/tile_ops.cc @@ -42,14 +42,14 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef Eigen::GpuDevice GPUDevice; #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL // Forward declarations of functors that will be defined in tile_ops_impl.h namespace functor { -template +template struct Tile { void operator()(const Device& d, Tensor* out, const Tensor& in, - const gtl::ArraySlice broadcast_array) const; + const gtl::ArraySlice broadcast_array) const; }; template @@ -80,7 +80,7 @@ struct ReduceAndReshape { } // namespace functor // -------------------------------------------------------------------------- -template +template class TileOp : public OpKernel { public: explicit TileOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -105,8 +105,8 @@ class TileOp : public OpKernel { return; } - const gtl::ArraySlice multiples_array(multiples.flat().data(), - input_dims); + const gtl::ArraySlice multiples_array( + multiples.flat().data(), input_dims); TensorShape output_shape; for (int i = 0; i < input_dims; ++i) { OP_REQUIRES( @@ -125,10 +125,10 @@ class TileOp : public OpKernel { // If there's no output, there's nothing to do. if (output_shape.num_elements() == 0) return; -#define HANDLE_TYPE(DT) \ - if (context->input(0).dtype() == DT) { \ - HandleCase
(context, multiples_array, result); \ - return; \ +#define HANDLE_TYPE(DT) \ + if (context->input(0).dtype() == DT) { \ + HandleCase
(context, multiples_array, result); \ + return; \ } #define HANDLE_TYPE_NAME(T) HANDLE_TYPE(DataTypeToEnum::value) @@ -158,27 +158,27 @@ class TileOp : public OpKernel { private: template void HandleCaseImpl(OpKernelContext* context, - const gtl::ArraySlice& multiples_array, + const gtl::ArraySlice& multiples_array, Tensor* result) { typedef typename EnumToDataType
::Type T; - functor::Tile() ( - context->eigen_device(), result, - context->input(0), multiples_array); + functor::Tile()(context->eigen_device(), + result, context->input(0), + multiples_array); } template void HandleCase(OpKernelContext* context, - const gtl::ArraySlice& multiples_array, + const gtl::ArraySlice& multiples_array, Tensor* result); TF_DISALLOW_COPY_AND_ASSIGN(TileOp); }; -template +template template -inline void TileOp::HandleCase( - OpKernelContext* context, const gtl::ArraySlice& multiples_array, - Tensor* result) { +inline void TileOp::HandleCase( + OpKernelContext* context, + const gtl::ArraySlice& multiples_array, Tensor* result) { // TODO(vrv): print out the device name if useful. Currently disabled to avoid // having to use RTTI. LOG(FATAL) << "TileOp: Invalid combination of Device, DT: " @@ -186,25 +186,28 @@ inline void TileOp::HandleCase( << DataTypeString(DT); } -#define HANDLE_CASE(device, dtype) \ - template <> \ - template <> \ - void TileOp::HandleCase( \ - OpKernelContext * context, \ - const gtl::ArraySlice& multiples_array, Tensor* result) { \ - HandleCaseImpl(context, multiples_array, result); \ +#define HANDLE_CASE(device, dtype, Tmultiples) \ + template <> \ + template <> \ + void TileOp::HandleCase( \ + OpKernelContext * context, \ + const gtl::ArraySlice& multiples_array, Tensor* result) { \ + HandleCaseImpl(context, multiples_array, result); \ } -#define HANDLE_TYPE_NAME_CPU(T) \ - HANDLE_CASE(CPUDevice, DataTypeToEnum::value); +#define HANDLE_TYPE_NAME_CPU(T) \ + HANDLE_CASE(CPUDevice, DataTypeToEnum::value, int32); \ + HANDLE_CASE(CPUDevice, DataTypeToEnum::value, int64); -#define HANDLE_TYPE_NAME_GPU(T) \ - HANDLE_CASE(GPUDevice, DataTypeToEnum::value); +#define HANDLE_TYPE_NAME_GPU(T) \ + HANDLE_CASE(GPUDevice, DataTypeToEnum::value, int32); \ + HANDLE_CASE(GPUDevice, DataTypeToEnum::value, int64); #ifdef TENSORFLOW_USE_SYCL -#define HANDLE_TYPE_NAME_SYCL(T) \ - HANDLE_CASE(SYCLDevice, DataTypeToEnum::value); -#endif // TENSORFLOW_USE_SYCL +#define HANDLE_TYPE_NAME_SYCL(T) \ + HANDLE_CASE(SYCLDevice, DataTypeToEnum::value, int32); \ + HANDLE_CASE(SYCLDevice, DataTypeToEnum::value, int64); +#endif // TENSORFLOW_USE_SYCL TF_CALL_bool(HANDLE_TYPE_NAME_CPU); TF_CALL_float(HANDLE_TYPE_NAME_CPU); @@ -235,17 +238,17 @@ TF_CALL_double(HANDLE_TYPE_NAME_SYCL); TF_CALL_int16(HANDLE_TYPE_NAME_SYCL); TF_CALL_int32(HANDLE_TYPE_NAME_SYCL); TF_CALL_int64(HANDLE_TYPE_NAME_SYCL); -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL #undef HANDLE_TYPE_NAME_CPU #undef HANDLE_TYPE_NAME_GPU #ifdef TENSORFLOW_USE_SYCL #undef HANDLE_TYPE_NAME_SYCL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL #undef HANDLE_CASE // -------------------------------------------------------------------------- -template +template class TileGradientOp : public OpKernel { public: explicit TileGradientOp(OpKernelConstruction* context) : OpKernel(context) {} @@ -270,10 +273,10 @@ class TileGradientOp : public OpKernel { return; } - const gtl::ArraySlice multiples_array(multiples.flat().data(), - input_dims); + const gtl::ArraySlice multiples_array( + multiples.flat().data(), input_dims); TensorShape output_shape; - std::vector input_dim_size_vec; + std::vector input_dim_size_vec; for (int i = 0; i < input_dims; ++i) { OP_REQUIRES( context, multiples_array[i] > 0, @@ -334,19 +337,19 @@ class TileGradientOp : public OpKernel { private: template void HandleCase(OpKernelContext* context, - const std::vector& input_dims, - const gtl::ArraySlice& multiples_array, + const std::vector& input_dims, + const gtl::ArraySlice& multiples_array, Tensor* result); template void HandleCaseImpl(OpKernelContext* context, - const std::vector& input_dims, - const gtl::ArraySlice& multiples_array, + const std::vector& input_dims, + const gtl::ArraySlice& multiples_array, Tensor* result) { typedef typename EnumToDataType
::Type T; bool reduction_only = true; - std::vector reduction_dims; + std::vector reduction_dims; for (int i = 0; i < NDIM; ++i) { if (input_dims[i] > multiples_array[i] && multiples_array[i] > 1) { @@ -408,7 +411,8 @@ class TileGradientOp : public OpKernel { template void HandleReduce(OpKernelContext* context, - const std::vector& reduce_dim_in, Tensor* result) { + const std::vector& reduce_dim_in, + Tensor* result) { static_assert(NDIM >= REDUCENDIM, "Too many reduced dimensions"); Eigen::DSizes reduce_dim; Eigen::DSizes reshape_dim; @@ -429,34 +433,41 @@ class TileGradientOp : public OpKernel { TF_DISALLOW_COPY_AND_ASSIGN(TileGradientOp); }; -template +template template -inline void TileGradientOp::HandleCase( - OpKernelContext* context, const std::vector& input_dims, - const gtl::ArraySlice& multiples_array, Tensor* result) { +inline void TileGradientOp::HandleCase( + OpKernelContext* context, const std::vector& input_dims, + const gtl::ArraySlice& multiples_array, Tensor* result) { LOG(FATAL) << "TileGradientOp: Invalid combination of Device, DT and NDIM: " << MakeTypeIndex().name() << ", " << DataTypeString(DT) << ", " << NDIM; } -#define HANDLE_CASE(device, T, dtype, ndim) \ +#define HANDLE_CASE(device, T, dtype, Tmultiples, ndim) \ template <> \ template <> \ - void TileGradientOp::HandleCase( \ - OpKernelContext * context, const std::vector& input_dims, \ - const gtl::ArraySlice& multiples_array, Tensor* result) { \ + void TileGradientOp::HandleCase( \ + OpKernelContext * context, const std::vector& input_dims, \ + const gtl::ArraySlice& multiples_array, Tensor* result) { \ HandleCaseImpl(context, input_dims, multiples_array, result); \ } // 0-D handled specially above -#define HANDLE_CASE_DIM(device, T, dtype) \ - HANDLE_CASE(device, T, dtype, 1); \ - HANDLE_CASE(device, T, dtype, 2); \ - HANDLE_CASE(device, T, dtype, 3); \ - HANDLE_CASE(device, T, dtype, 4); \ - HANDLE_CASE(device, T, dtype, 5); \ - HANDLE_CASE(device, T, dtype, 6); \ - HANDLE_CASE(device, T, dtype, 7); +#define HANDLE_CASE_DIM(device, T, dtype) \ + HANDLE_CASE(device, T, dtype, int32, 1); \ + HANDLE_CASE(device, T, dtype, int32, 2); \ + HANDLE_CASE(device, T, dtype, int32, 3); \ + HANDLE_CASE(device, T, dtype, int32, 4); \ + HANDLE_CASE(device, T, dtype, int32, 5); \ + HANDLE_CASE(device, T, dtype, int32, 6); \ + HANDLE_CASE(device, T, dtype, int32, 7); \ + HANDLE_CASE(device, T, dtype, int64, 1); \ + HANDLE_CASE(device, T, dtype, int64, 2); \ + HANDLE_CASE(device, T, dtype, int64, 3); \ + HANDLE_CASE(device, T, dtype, int64, 4); \ + HANDLE_CASE(device, T, dtype, int64, 5); \ + HANDLE_CASE(device, T, dtype, int64, 6); \ + HANDLE_CASE(device, T, dtype, int64, 7); #define HANDLE_TYPE_NAME_CPU(T) \ HANDLE_CASE_DIM(CPUDevice, T, DataTypeToEnum::value); @@ -494,7 +505,7 @@ TF_CALL_int16(HANDLE_TYPE_NAME_SYCL); TF_CALL_int32(HANDLE_TYPE_NAME_SYCL); TF_CALL_int64(HANDLE_TYPE_NAME_SYCL); #undef HANDLE_TYPE_NAME_SYCL -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL #undef HANDLE_TYPE_NAME_CPU #undef HANDLE_TYPE_NAME_GPU @@ -505,127 +516,92 @@ REGISTER_KERNEL_BUILDER(Name("Tile") .Device(DEVICE_CPU) .HostMemory("multiples") .TypeConstraint("Tmultiples"), - TileOp); -REGISTER_KERNEL_BUILDER( - Name("TileGrad").Device(DEVICE_CPU).HostMemory("multiples"), - TileGradientOp); - -#if GOOGLE_CUDA - -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); + TileOp); REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); - -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); + .Device(DEVICE_CPU) + .HostMemory("multiples") + .TypeConstraint("Tmultiples"), + TileOp); REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); + .Device(DEVICE_CPU) + .HostMemory("multiples") + .TypeConstraint("Tmultiples"), + TileGradientOp); REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_GPU) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); + .Device(DEVICE_CPU) + .HostMemory("multiples") + .TypeConstraint("Tmultiples"), + TileGradientOp); +#if GOOGLE_CUDA +#define REGISTER_GPU(type) \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); + +TF_CALL_float(REGISTER_GPU); +TF_CALL_double(REGISTER_GPU); +TF_CALL_half(REGISTER_GPU); +TF_CALL_int16(REGISTER_GPU); +TF_CALL_int32(REGISTER_GPU); +TF_CALL_complex64(REGISTER_GPU); +TF_CALL_complex128(REGISTER_GPU) + +#undef REGISTER_GPU #endif // GOOGLE_CUDA #ifdef TENSORFLOW_USE_SYCL -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); -REGISTER_KERNEL_BUILDER(Name("Tile") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileOp); - -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -REGISTER_KERNEL_BUILDER(Name("TileGrad") - .Device(DEVICE_SYCL) - .TypeConstraint("T") - .TypeConstraint("Tmultiples") - .HostMemory("multiples"), - TileGradientOp); -#endif // TENSORFLOW_USE_SYCL +#define REGISTER_SYCL(type) \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("Tile") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); \ + REGISTER_KERNEL_BUILDER(Name("TileGrad") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .TypeConstraint("Tmultiples") \ + .HostMemory("multiples"), \ + TileGradientOp); + + TF_CALL_float(REGISTER_SYCL); +TF_CALL_double(REGISTER_SYCL); + +#undef REGISTER_SYCL +#endif // TENSORFLOW_USE_SYCL } // namespace tensorflow diff --git a/tensorflow/core/kernels/topk_op_gpu.cu.cc b/tensorflow/core/kernels/topk_op_gpu.cu.cc index 10a7602dc4696636be9228e101ef641cf3389158..ca296d5aa044d6818be85a64fe297cf7974d909b 100644 --- a/tensorflow/core/kernels/topk_op_gpu.cu.cc +++ b/tensorflow/core/kernels/topk_op_gpu.cu.cc @@ -379,7 +379,7 @@ cudaError LaunchTopKKernel(const cudaStream_t& stream, int num_shards, // Use as many shards as possible. if (num_shards <= 0) { constexpr auto shared_memory_size = 48 << 10; // 48 KB - const auto heap_size = k * (sizeof(int) + sizeof(T)); + const auto heap_size = k * sizeof(Entry); // shared_memory_size = (num_shards + 1) * heap_size <=> num_shards = shared_memory_size / heap_size - 1; if (num_shards <= 0) { diff --git a/tensorflow/core/kernels/transpose_functor.h b/tensorflow/core/kernels/transpose_functor.h index 87569f0275bfe1ffe2283ffbd8382cde516531b8..add4635331ee55495f5bc0d79fd040812078f1f8 100644 --- a/tensorflow/core/kernels/transpose_functor.h +++ b/tensorflow/core/kernels/transpose_functor.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_ #define TENSORFLOW_CORE_KERNELS_TRANSPOSE_FUNCTOR_H_ +#include #include #include #include "tensorflow/core/framework/tensor.h" @@ -23,7 +24,6 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" namespace tensorflow { - // Transpose tensor 'in' into tensor 'out' according to dimension // permutation 'perm'. // @@ -46,6 +46,17 @@ template Status DoConjugateTranspose(const Device& device, const Tensor& in, const gtl::ArraySlice perm, Tensor* out); +// Convenience versions of DoTranspose that only swap the last (inner) two +// dimensions. +template +Status DoMatrixTranspose(const Device& device, const Tensor& in, Tensor* out); + +// Convenience versions of DoConjugateTranspose that only swap the last (inner) +// two dimensions. +template +Status DoConjugateMatrixTranspose(const Device& device, const Tensor& in, + Tensor* out); + // Primary device specific functor to be specialized for each device and type. template struct Transpose { @@ -131,11 +142,6 @@ inline bool NonSingletonDimensionsAlign(const TensorShape& input_shape, return true; } -// Device-specific naive implementation for transpose. -template -void TransposeSimple(const Device& d, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out); - // Uses Eigen to transpose. template void TransposeUsingEigen(const Device& d, const Tensor& in, @@ -157,69 +163,87 @@ void TransposeUsingEigen(const Device& d, const Tensor& in, } template -struct DoTransposeImpl { - static Status run(const Device& d, const Tensor& in, - const gtl::ArraySlice perm, bool conjugate, - Tensor* out) { - CHECK_GE(in.dims(), 2); - CHECK_EQ(in.dims(), out->dims()); - CHECK_EQ(in.dims(), perm.size()); - CHECK_EQ(in.dtype(), out->dtype()); - switch (in.dtype()) { - case DT_BOOL: - case DT_INT8: - case DT_QINT8: - case DT_QUINT8: - case DT_UINT8: - Transpose::run(d, in, perm, out); - break; - - case DT_BFLOAT16: - case DT_HALF: - case DT_INT16: - case DT_QINT16: - case DT_QUINT16: - case DT_UINT16: - Transpose::run(d, in, perm, out); - break; - - case DT_FLOAT: - case DT_INT32: - case DT_QINT32: - Transpose::run(d, in, perm, out); - break; - - case DT_DOUBLE: - case DT_INT64: +Status DoTransposeImpl(const Device& d, const Tensor& in, + const gtl::ArraySlice perm, bool conjugate, + Tensor* out) { + CHECK_GE(in.dims(), 2); + CHECK_EQ(in.dims(), out->dims()); + CHECK_EQ(in.dims(), perm.size()); + CHECK_EQ(in.dtype(), out->dtype()); + switch (in.dtype()) { + case DT_BOOL: + case DT_INT8: + case DT_QINT8: + case DT_QUINT8: + case DT_UINT8: + Transpose::run(d, in, perm, out); + break; + + case DT_BFLOAT16: + case DT_HALF: + case DT_INT16: + case DT_QINT16: + case DT_QUINT16: + case DT_UINT16: + Transpose::run(d, in, perm, out); + break; + + case DT_FLOAT: + case DT_INT32: + case DT_QINT32: + Transpose::run(d, in, perm, out); + break; + + case DT_DOUBLE: + case DT_INT64: + Transpose::run(d, in, perm, out); + break; + + case DT_COMPLEX64: + if (conjugate) { +#if defined(__ANDROID__) and !defined(__clang__) + // Workaround for GCC compiler bug in Android toolchain. + return errors::Unimplemented( + "Conjugate transpose of complex64 not supported for GCC on " + "Android."); +#else + Transpose::run(d, in, perm, out); +#endif + } else { Transpose::run(d, in, perm, out); - break; - - case DT_COMPLEX64: - if (conjugate) { - Transpose::run(d, in, perm, out); - } else { - Transpose::run(d, in, perm, out); - } - break; - - case DT_COMPLEX128: - if (conjugate) { - Transpose::run(d, in, perm, out); - } else { - Transpose::run(d, in, perm, out); - } - break; - - case DT_STRING: - Transpose::run(d, in, perm, out); - break; - - default: - return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype()); - } - return Status::OK(); + } + break; + + case DT_COMPLEX128: + if (conjugate) { + Transpose::run(d, in, perm, + out); + } else { + Transpose::run(d, in, perm, + out); + } + break; + + case DT_STRING: + Transpose::run(d, in, perm, out); + break; + + default: + return errors::Unimplemented("Unsupported dtype on CPU: ", in.dtype()); } -}; + return Status::OK(); +} + +template +inline Status DoMatrixTransposeImpl(const Device& device, const Tensor& in, + bool conjugate, Tensor* out) { + const int ndims = in.dims(); + if (ndims == 0) return Status::OK(); + TransposePermsVec perm(ndims); + std::iota(perm.begin(), perm.end(), 0); + std::swap(perm[ndims - 2], perm[ndims - 1]); + return DoTransposeImpl(device, in, perm, conjugate, out); +} #ifdef TENSORFLOW_USE_SYCL // For SYCL lets always go through Eigen diff --git a/tensorflow/core/kernels/transpose_functor_cpu.cc b/tensorflow/core/kernels/transpose_functor_cpu.cc index b2de012be1dcf3921788b5e9114c75395ee8397e..41b73fdaf4aced13070164afb81825592637f8c4 100644 --- a/tensorflow/core/kernels/transpose_functor_cpu.cc +++ b/tensorflow/core/kernels/transpose_functor_cpu.cc @@ -29,17 +29,18 @@ limitations under the License. typedef Eigen::ThreadPoolDevice CPUDevice; namespace tensorflow { -namespace internal { +namespace { -template -void TransposeSimple(const Device& device, const Tensor& in, +template +void TransposeSimple(const CPUDevice& device, const Tensor& in, const gtl::ArraySlice perm, Tensor* out) { const int ndims = in.dims(); gtl::InlinedVector in_strides = ComputeStride(in.shape()); gtl::InlinedVector out_strides = ComputeStride(out->shape()); const T* p = reinterpret_cast(in.tensor_data().data()); T* q = reinterpret_cast(const_cast((out->tensor_data().data()))); - auto transpose_fn = [=](int64 begin, int64 end) { + auto transpose_fn = [=, &in_strides, &out_strides, &perm](int64 begin, + int64 end) { for (int64 o_idx = begin; o_idx < end; ++o_idx) { int64 i_idx = 0; int64 t = o_idx; @@ -64,7 +65,7 @@ void TransposeSimple(const Device& device, const Tensor& in, device.parallelFor(in.NumElements(), cost, std::move(transpose_fn)); } -} // end namespace internal +} // namespace template struct Transpose { @@ -88,32 +89,47 @@ struct Transpose { out); break; default: - internal::TransposeSimple(d, in, perm, out); + TransposeSimple(d, in, perm, out); break; } } }; -template <> -Status DoTranspose(const CPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - false /* conjugate */, out); -} +#define INSTANTIATE(DEVICE) \ + template <> \ + Status DoTranspose(const DEVICE& device, const Tensor& in, \ + const gtl::ArraySlice perm, Tensor* out) { \ + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/false, \ + out); \ + } \ + template <> \ + Status DoConjugateTranspose(const DEVICE& device, const Tensor& in, \ + const gtl::ArraySlice perm, \ + Tensor* out) { \ + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/true, \ + out); \ + } \ + template <> \ + Status DoMatrixTranspose(const DEVICE& device, const Tensor& in, \ + Tensor* out) { \ + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/false, \ + out); \ + } \ + template <> \ + Status DoConjugateMatrixTranspose(const DEVICE& device, const Tensor& in, \ + Tensor* out) { \ + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/true, \ + out); \ + } -template <> -Status DoConjugateTranspose(const CPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - true /* conjugate */, out); -} +INSTANTIATE(CPUDevice) #ifdef TENSORFLOW_USE_SYCL typedef Eigen::SyclDevice SYCLDevice; namespace internal { -template -void TransposeSYCL(const Device& d, const Tensor& in, +template +void TransposeSYCL(const SYCLDevice& d, const Tensor& in, const gtl::ArraySlice perm, bool conjugate, Tensor* out) { switch (in.dims()) { @@ -165,19 +181,11 @@ struct Transpose { } }; -template <> -Status DoTranspose(const SYCLDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - false /* conjugate */, out); -} +// Explicit instantiation. +template struct Transpose; -template <> -Status DoConjugateTranspose(const SYCLDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - true /* conjugate */, out); -} +INSTANTIATE(SYCLDevice) +#undef INSTANTIATE #endif // TENSORFLOW_USE_SYCL diff --git a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc index 364baf9a513a862e0b76b253bf70b24047347776..493dac9a7ca5a57dba10a3c155299d78e3a69f38 100644 --- a/tensorflow/core/kernels/transpose_functor_gpu.cu.cc +++ b/tensorflow/core/kernels/transpose_functor_gpu.cu.cc @@ -53,8 +53,8 @@ __global__ void TransposeKernel(int nthreads, const T* src, const int32* buf, } } -template -void TransposeSimple(const Device& d, const Tensor& in, +template +void TransposeSimple(const GPUDevice& d, const Tensor& in, const gtl::ArraySlice perm, Tensor* out) { // Ensures we can use 32-bit index. const int64 nelem = in.NumElements(); @@ -165,23 +165,9 @@ struct TransposeUsingTile { } }; -} // end namespace internal - -template <> -Status DoTranspose(const GPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - false /* conjugate */, out); -} - -template <> -Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in, - const gtl::ArraySlice perm, Tensor* out) { - return internal::DoTransposeImpl::run(device, in, perm, - true /* conjugate */, out); -} +} // namespace internal -// Transpose kernel specialized for CPU Device. +// Transpose kernel specialized for GPU Device. template struct Transpose { static void run(const GPUDevice& d, const Tensor& in, @@ -216,19 +202,43 @@ struct Transpose { } break; default: - internal::TransposeSimple(d, in, perm, out); + internal::TransposeSimple(d, in, perm, out); break; } } }; -template <> -struct Transpose { +template +struct Transpose { static void run(const GPUDevice& d, const Tensor& in, const gtl::ArraySlice perm, Tensor* out) { LOG(FATAL) << "Transpose of DT_STRING tensor not supported on GPU."; } }; +// Explicit instantiation. +template struct Transpose; + +template <> +Status DoTranspose(const GPUDevice& device, const Tensor& in, + const gtl::ArraySlice perm, Tensor* out) { + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/false, out); +} +template <> +Status DoConjugateTranspose(const GPUDevice& device, const Tensor& in, + const gtl::ArraySlice perm, Tensor* out) { + return internal::DoTransposeImpl(device, in, perm, /*conjugate=*/true, out); +} +template <> +Status DoMatrixTranspose(const GPUDevice& device, const Tensor& in, + Tensor* out) { + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/false, out); +} +template <> +Status DoConjugateMatrixTranspose(const GPUDevice& device, const Tensor& in, + Tensor* out) { + return internal::DoMatrixTransposeImpl(device, in, /*conjugate=*/true, out); +} + } // namespace tensorflow #endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/transpose_op.cc b/tensorflow/core/kernels/transpose_op.cc index e151b38d90a60b18ffaeaad56f132706df2eb3a6..20f0edf309a0a61f306ebc6321577830203f7764 100644 --- a/tensorflow/core/kernels/transpose_op.cc +++ b/tensorflow/core/kernels/transpose_op.cc @@ -91,6 +91,26 @@ REGISTER_KERNEL_BUILDER(Name("InvertPermutation") InvertPermutationOp); #endif // TENSORFLOW_USE_SYCL +namespace { +template +Status PermutationHelper(const Tensor& perm, const int dims, + std::vector* permutation) { + auto Vperm = perm.vec(); + if (dims != Vperm.size()) { + return errors::InvalidArgument("transpose expects a vector of size ", dims, + ". But input(1) is a vector of size ", + Vperm.size()); + } + // using volatile instead of SubtleMustCopy here so that the + // asynchrony boundary is permutation. + const volatile Tperm* perm_begin = + reinterpret_cast(Vperm.data()); + *permutation = std::vector(perm_begin, perm_begin + dims); + + return Status::OK(); +} +} // namespace + // output = TransposeOp(T input, T perm) takes a tensor // of type T and rank N, and a permutation of 0, 1, ..., N-1. It // shuffles the dimensions of the input tensor according to permutation. @@ -113,17 +133,16 @@ void TransposeOp::Compute(OpKernelContext* ctx) { OP_REQUIRES(ctx, TensorShapeUtils::IsVector(perm.shape()), errors::InvalidArgument("perm must be a vector, not ", perm.shape().DebugString())); - auto Vperm = perm.vec(); + + // Although Tperm may be an int64 type, an int32 is sufficient to hold + // dimension range values, so the narrowing here should be safe. + std::vector permutation; const int dims = input.dims(); - OP_REQUIRES(ctx, dims == Vperm.size(), - errors::InvalidArgument( - "transpose expects a vector of size ", input.dims(), - ". But input(1) is a vector of size ", Vperm.size())); - // using volatile instead of SubtleMustCopy here so that the - // asynchrony boundary is permutation. - const volatile int32* perm_begin = - reinterpret_cast(Vperm.data()); - const std::vector permutation(perm_begin, perm_begin + dims); + if (perm.dtype() == DT_INT32) { + OP_REQUIRES_OK(ctx, PermutationHelper(perm, dims, &permutation)); + } else { + OP_REQUIRES_OK(ctx, PermutationHelper(perm, dims, &permutation)); + } TensorShape shape; // Check whether permutation is a permutation of integers of [0 .. dims). @@ -142,10 +161,9 @@ void TransposeOp::Compute(OpKernelContext* ctx) { } } for (int i = 0; i < dims; ++i) { - OP_REQUIRES( - ctx, bits[i], - errors::InvalidArgument(i, " is missing from {", - str_util::Join(permutation, ","), "}.")); + OP_REQUIRES(ctx, bits[i], errors::InvalidArgument( + i, " is missing from {", + str_util::Join(permutation, ","), "}.")); } // 0-D, 1-D, and identity transposes do nothing. @@ -185,18 +203,16 @@ Status ConjugateTransposeCpuOp::DoTranspose(OpKernelContext* ctx, } #ifdef INTEL_MKL -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - MklTransposeCpuOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + MklTransposeCpuOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ MklConjugateTransposeCpuOp); TF_CALL_ALL_TYPES(REGISTER); REGISTER(bfloat16); @@ -204,18 +220,16 @@ REGISTER(bfloat16); #else // INTEL_MKL -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - TransposeCpuOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + TransposeCpuOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ ConjugateTransposeCpuOp); TF_CALL_ALL_TYPES(REGISTER) REGISTER(bfloat16); @@ -238,18 +252,16 @@ Status ConjugateTransposeGpuOp::DoTranspose(OpKernelContext* ctx, perm, out); } -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - TransposeGpuOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_GPU) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + TransposeGpuOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_GPU) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ ConjugateTransposeGpuOp); TF_CALL_POD_TYPES(REGISTER); #undef REGISTER @@ -270,18 +282,16 @@ Status ConjugateTransposeSyclOp::DoTranspose(OpKernelContext* ctx, return ::tensorflow::DoConjugateTranspose(ctx->eigen_device(), in, perm, out); } -#define REGISTER(T) \ - REGISTER_KERNEL_BUILDER(Name("Transpose") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ - TransposeSyclOp); \ - REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ - .Device(DEVICE_SYCL) \ - .TypeConstraint("T") \ - .TypeConstraint("Tperm") \ - .HostMemory("perm"), \ +#define REGISTER(T) \ + REGISTER_KERNEL_BUILDER(Name("Transpose") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ + TransposeSyclOp); \ + REGISTER_KERNEL_BUILDER(Name("ConjugateTranspose") \ + .Device(DEVICE_SYCL) \ + .TypeConstraint("T") \ + .HostMemory("perm"), \ ConjugateTransposeSyclOp); TF_CALL_POD_TYPES(REGISTER); #undef REGISTER diff --git a/tensorflow/core/kernels/transpose_op.h b/tensorflow/core/kernels/transpose_op.h index ff9cf5d4fffb497549787362fd5863f56e3eef6d..ae67592d044f9ebd67905641d51df780b261489f 100644 --- a/tensorflow/core/kernels/transpose_op.h +++ b/tensorflow/core/kernels/transpose_op.h @@ -86,7 +86,6 @@ class ConjugateTransposeCpuOp : public TransposeOp { }; #ifdef INTEL_MKL -template class MklConjugateTransposeCpuOp : public TransposeOp { public: explicit MklConjugateTransposeCpuOp(OpKernelConstruction* ctx) diff --git a/tensorflow/core/lib/core/coding.h b/tensorflow/core/lib/core/coding.h index 77d52a909baa6c1795a753ee26f51cc3c17d9dc8..8265aec8703489c2c6e008cfca8af3072fdc9bc0 100644 --- a/tensorflow/core/lib/core/coding.h +++ b/tensorflow/core/lib/core/coding.h @@ -31,6 +31,9 @@ namespace core { // Maximum number of bytes occupied by a varint32. static const int kMaxVarint32Bytes = 5; +// Maximum number of bytes occupied by a varint64. +static const int kMaxVarint64Bytes = 10; + // Lower-level versions of Put... that write directly into a character buffer // REQUIRES: dst has enough space for the value being written extern void EncodeFixed16(char* dst, uint16 value); diff --git a/tensorflow/core/lib/db/BUILD b/tensorflow/core/lib/db/BUILD index 367686c16a86de544e1ae03109d1a6b69208007e..41b7af1b6993d967370e54f080fcd63a4483d4b6 100644 --- a/tensorflow/core/lib/db/BUILD +++ b/tensorflow/core/lib/db/BUILD @@ -12,6 +12,7 @@ cc_library( srcs = ["sqlite.cc"], hdrs = ["sqlite.h"], deps = [ + "//tensorflow/compiler/xla:statusor", "//tensorflow/core:lib", "@sqlite_archive//:sqlite", ], diff --git a/tensorflow/core/lib/db/sqlite.cc b/tensorflow/core/lib/db/sqlite.cc index 108be452a221cedc7397fb8ebe31629464ad34d8..701655f622a7ec0288f1cb53818877e65839643e 100644 --- a/tensorflow/core/lib/db/sqlite.cc +++ b/tensorflow/core/lib/db/sqlite.cc @@ -18,14 +18,13 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" namespace tensorflow { -namespace db { /* static */ -Status Sqlite::Open(const string& uri, std::unique_ptr* db) { +xla::StatusOr> Sqlite::Open(const string& uri) { sqlite3* sqlite = nullptr; Status s = MakeStatus(sqlite3_open(uri.c_str(), &sqlite)); if (s.ok()) { - *db = std::unique_ptr(new Sqlite(sqlite)); + return std::shared_ptr(new Sqlite(sqlite)); } return s; } @@ -87,6 +86,9 @@ Sqlite::~Sqlite() { } Status Sqlite::Close() { + if (db_ == nullptr) { + return Status::OK(); + } // If Close is explicitly called, ordering must be correct. Status s = MakeStatus(sqlite3_close(db_)); if (s.ok()) { @@ -95,23 +97,42 @@ Status Sqlite::Close() { return s; } -std::unique_ptr Sqlite::Prepare(const string& sql) { +SqliteStatement Sqlite::Prepare(const string& sql) { sqlite3_stmt* stmt = nullptr; int rc = sqlite3_prepare_v2(db_, sql.c_str(), sql.size() + 1, &stmt, nullptr); - return std::unique_ptr(new SqliteStatement(stmt, rc)); + if (rc == SQLITE_OK) { + return {stmt, SQLITE_OK, std::unique_ptr(nullptr)}; + } else { + return {nullptr, rc, std::unique_ptr(new string(sql))}; + } } -SqliteStatement::SqliteStatement(sqlite3_stmt* stmt, int error) - : stmt_(stmt), error_(error) {} +Status SqliteStatement::status() const { + Status s = Sqlite::MakeStatus(error_); + if (!s.ok()) { + if (stmt_ != nullptr) { + errors::AppendToMessage(&s, sqlite3_sql(stmt_)); + } else { + errors::AppendToMessage(&s, *prepare_error_sql_); + } + } + return s; +} -SqliteStatement::~SqliteStatement() { - int rc = sqlite3_finalize(stmt_); - if (rc != SQLITE_OK) { - LOG(ERROR) << "destruct sqlite3_stmt: " << Sqlite::MakeStatus(rc); +void SqliteStatement::CloseOrLog() { + if (stmt_ != nullptr) { + int rc = sqlite3_finalize(stmt_); + if (rc != SQLITE_OK) { + LOG(ERROR) << "destruct sqlite3_stmt: " << Sqlite::MakeStatus(rc); + } + stmt_ = nullptr; } } Status SqliteStatement::Close() { + if (stmt_ == nullptr) { + return Status::OK(); + } int rc = sqlite3_finalize(stmt_); if (rc == SQLITE_OK) { stmt_ = nullptr; @@ -121,8 +142,10 @@ Status SqliteStatement::Close() { } void SqliteStatement::Reset() { - sqlite3_reset(stmt_); - sqlite3_clear_bindings(stmt_); + if (TF_PREDICT_TRUE(stmt_ != nullptr)) { + sqlite3_reset(stmt_); + sqlite3_clear_bindings(stmt_); // not nullptr friendly + } error_ = SQLITE_OK; } @@ -163,5 +186,4 @@ Status SqliteStatement::StepAndReset() { return s; } -} // namespace db } // namespace tensorflow diff --git a/tensorflow/core/lib/db/sqlite.h b/tensorflow/core/lib/db/sqlite.h index 316e938f1b8337dab2e083b90cf43cb724bb1cc8..774852efea7b494406c89960654b1acdca1f4ac9 100644 --- a/tensorflow/core/lib/db/sqlite.h +++ b/tensorflow/core/lib/db/sqlite.h @@ -17,15 +17,16 @@ limitations under the License. #include #include +#include #include "sqlite3.h" +#include "tensorflow/compiler/xla/statusor.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { -namespace db { class SqliteStatement; @@ -46,7 +47,7 @@ class Sqlite { /// `file::memory:` for testing. /// /// See https://sqlite.org/c3ref/open.html - static Status Open(const string& uri, std::unique_ptr* db); + static xla::StatusOr> Open(const string& uri); /// \brief Makes tensorflow::Status for SQLite result code. /// @@ -65,7 +66,7 @@ class Sqlite { /// \brief Frees underlying SQLite object. /// /// Unlike the destructor, all SqliteStatement objects must be closed - /// beforehand. + /// beforehand. This is a no-op if already closed Status Close(); /// \brief Creates SQLite statement. @@ -74,7 +75,7 @@ class Sqlite { /// failed. It is also possible to punt the error checking to after /// the values have been binded and Step() or ExecuteWriteQuery() is /// called. - std::unique_ptr Prepare(const string& sql); + SqliteStatement Prepare(const string& sql); private: explicit Sqlite(sqlite3* db); @@ -89,21 +90,34 @@ class Sqlite { /// Instances of this class are not thread safe. class SqliteStatement { public: - /// \brief Destroys object and finalizes statement if needed. - ~SqliteStatement(); + /// \brief Constructs empty statement that should be assigned later. + SqliteStatement() : stmt_(nullptr), error_(SQLITE_OK) {} + + /// \brief Empties object and finalizes statement if needed. + ~SqliteStatement() { CloseOrLog(); } + + /// \brief Move constructor, after which should not be used. + SqliteStatement(SqliteStatement&& other); + + /// \brief Move assignment, after which should not be used. + SqliteStatement& operator=(SqliteStatement&& other); + + /// \brief Returns true if statement is not empty. + operator bool() const { return stmt_ != nullptr; } /// \brief Returns SQLite result code state. /// /// This will be SQLITE_OK unless an error happened. If multiple /// errors happened, only the first error code will be returned. - int error() { return error_; } + int error() const { return error_; } /// \brief Returns error() as a tensorflow::Status. - Status status() { return Sqlite::MakeStatus(error_); } + Status status() const; /// \brief Finalize statement object. /// - /// Please note that the destructor can also do this. + /// Please note that the destructor can also do this. This method is + /// a no-op if already closed. Status Close(); /// \brief Executes query and/or fetches next row. @@ -247,7 +261,12 @@ class SqliteStatement { private: friend Sqlite; - SqliteStatement(sqlite3_stmt* stmt, int error); // takes ownership + SqliteStatement(sqlite3_stmt* stmt, int error, + std::unique_ptr prepare_error_sql) + : stmt_(stmt), + error_(error), + prepare_error_sql_(std::move(prepare_error_sql)) {} + void CloseOrLog(); void Update(int rc) { if (TF_PREDICT_FALSE(rc != SQLITE_OK)) { @@ -268,11 +287,31 @@ class SqliteStatement { sqlite3_stmt* stmt_; int error_; + std::unique_ptr prepare_error_sql_; TF_DISALLOW_COPY_AND_ASSIGN(SqliteStatement); }; -} // namespace db +inline SqliteStatement::SqliteStatement(SqliteStatement&& other) + : stmt_(other.stmt_), + error_(other.error_), + prepare_error_sql_(std::move(other.prepare_error_sql_)) { + other.stmt_ = nullptr; + other.error_ = SQLITE_OK; +} + +inline SqliteStatement& SqliteStatement::operator=(SqliteStatement&& other) { + if (&other != this) { + CloseOrLog(); + stmt_ = other.stmt_; + error_ = other.error_; + prepare_error_sql_ = std::move(other.prepare_error_sql_); + other.stmt_ = nullptr; + other.error_ = SQLITE_OK; + } + return *this; +} + } // namespace tensorflow #endif // TENSORFLOW_CORE_LIB_DB_SQLITE_H_ diff --git a/tensorflow/core/lib/db/sqlite_test.cc b/tensorflow/core/lib/db/sqlite_test.cc index ce22379d97d928b3ea1a8ef06dfd2ca14d5f6320..ba045274adc605fbbaece7736537e8157e27cbc7 100644 --- a/tensorflow/core/lib/db/sqlite_test.cc +++ b/tensorflow/core/lib/db/sqlite_test.cc @@ -24,97 +24,96 @@ limitations under the License. #include "tensorflow/core/platform/test.h" namespace tensorflow { -namespace db { namespace { class SqliteTest : public ::testing::Test { protected: void SetUp() override { - TF_ASSERT_OK(Sqlite::Open(":memory:", &db_)); + db_ = Sqlite::Open(":memory:").ValueOrDie(); auto stmt = db_->Prepare("CREATE TABLE T (a BLOB, b BLOB)"); - TF_ASSERT_OK(stmt->StepAndReset()); + TF_ASSERT_OK(stmt.StepAndReset()); } - std::unique_ptr db_; + std::shared_ptr db_; bool is_done_; }; TEST_F(SqliteTest, InsertAndSelectInt) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindInt(1, 3); - stmt->BindInt(2, -7); - TF_ASSERT_OK(stmt->StepAndReset()); - stmt->BindInt(1, 123); - stmt->BindInt(2, -123); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindInt(1, 3); + stmt.BindInt(2, -7); + TF_ASSERT_OK(stmt.StepAndReset()); + stmt.BindInt(1, 123); + stmt.BindInt(2, -123); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T ORDER BY b"); - TF_ASSERT_OK(stmt->Step(&is_done_)); + TF_ASSERT_OK(stmt.Step(&is_done_)); ASSERT_FALSE(is_done_); - EXPECT_EQ(123, stmt->ColumnInt(0)); - EXPECT_EQ(-123, stmt->ColumnInt(1)); - TF_ASSERT_OK(stmt->Step(&is_done_)); + EXPECT_EQ(123, stmt.ColumnInt(0)); + EXPECT_EQ(-123, stmt.ColumnInt(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); ASSERT_FALSE(is_done_); - EXPECT_EQ(3, stmt->ColumnInt(0)); - EXPECT_EQ(-7, stmt->ColumnInt(1)); - TF_ASSERT_OK(stmt->Step(&is_done_)); + EXPECT_EQ(3, stmt.ColumnInt(0)); + EXPECT_EQ(-7, stmt.ColumnInt(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); ASSERT_TRUE(is_done_); } TEST_F(SqliteTest, InsertAndSelectDouble) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindDouble(1, 6.28318530); - stmt->BindDouble(2, 1.61803399); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindDouble(1, 6.28318530); + stmt.BindDouble(2, 1.61803399); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(6.28318530, stmt->ColumnDouble(0)); - EXPECT_EQ(1.61803399, stmt->ColumnDouble(1)); - EXPECT_EQ(6, stmt->ColumnInt(0)); - EXPECT_EQ(1, stmt->ColumnInt(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(6.28318530, stmt.ColumnDouble(0)); + EXPECT_EQ(1.61803399, stmt.ColumnDouble(1)); + EXPECT_EQ(6, stmt.ColumnInt(0)); + EXPECT_EQ(1, stmt.ColumnInt(1)); } TEST_F(SqliteTest, NulCharsInString) { string s; // XXX: Want to write {2, '\0'} but not sure why not. s.append(static_cast(2), '\0'); auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlob(1, s); - stmt->BindText(2, s); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindBlob(1, s); + stmt.BindText(2, s); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(2, stmt->ColumnSize(0)); - EXPECT_EQ(2, stmt->ColumnString(0).size()); - EXPECT_EQ('\0', stmt->ColumnString(0).at(0)); - EXPECT_EQ('\0', stmt->ColumnString(0).at(1)); - EXPECT_EQ(2, stmt->ColumnSize(1)); - EXPECT_EQ(2, stmt->ColumnString(1).size()); - EXPECT_EQ('\0', stmt->ColumnString(1).at(0)); - EXPECT_EQ('\0', stmt->ColumnString(1).at(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(2, stmt.ColumnSize(0)); + EXPECT_EQ(2, stmt.ColumnString(0).size()); + EXPECT_EQ('\0', stmt.ColumnString(0).at(0)); + EXPECT_EQ('\0', stmt.ColumnString(0).at(1)); + EXPECT_EQ(2, stmt.ColumnSize(1)); + EXPECT_EQ(2, stmt.ColumnString(1).size()); + EXPECT_EQ('\0', stmt.ColumnString(1).at(0)); + EXPECT_EQ('\0', stmt.ColumnString(1).at(1)); } TEST_F(SqliteTest, Unicode) { string s = "要依法治国是赞美那些谁是公义的和惩罚恶人。 - 韩非"; auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlob(1, s); - stmt->BindText(2, s); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindBlob(1, s); + stmt.BindText(2, s); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(s, stmt->ColumnString(0)); - EXPECT_EQ(s, stmt->ColumnString(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(s, stmt.ColumnString(0)); + EXPECT_EQ(s, stmt.ColumnString(1)); } TEST_F(SqliteTest, StepAndResetClearsBindings) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindInt(1, 1); - stmt->BindInt(2, 123); - TF_ASSERT_OK(stmt->StepAndReset()); - stmt->BindInt(1, 2); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindInt(1, 1); + stmt.BindInt(2, 123); + TF_ASSERT_OK(stmt.StepAndReset()); + stmt.BindInt(1, 2); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT b FROM T ORDER BY a"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(123, stmt->ColumnInt(0)); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(SQLITE_NULL, stmt->ColumnType(0)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(123, stmt.ColumnInt(0)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(SQLITE_NULL, stmt.ColumnType(0)); } TEST_F(SqliteTest, CloseBeforeFinalizeFails) { @@ -128,71 +127,109 @@ TEST_F(SqliteTest, CloseBeforeFinalizeFails) { // is designed to carry the first error state forward to Step(). TEST_F(SqliteTest, ErrorPuntingDoesNotReportLibraryAbuse) { auto stmt = db_->Prepare("lol cat"); - EXPECT_FALSE(stmt->status().ok()); - EXPECT_EQ(SQLITE_ERROR, stmt->error()); - stmt->BindInt(1, 1); - stmt->BindInt(2, 2); - Status s = stmt->Step(&is_done_); - EXPECT_EQ(SQLITE_ERROR, stmt->error()); // first error of several + EXPECT_FALSE(stmt.status().ok()); + EXPECT_EQ(SQLITE_ERROR, stmt.error()); + stmt.BindInt(1, 1); + stmt.BindInt(2, 2); + Status s = stmt.Step(&is_done_); + EXPECT_EQ(SQLITE_ERROR, stmt.error()); // first error of several EXPECT_FALSE(s.ok()); } TEST_F(SqliteTest, SafeBind) { string s = "hello"; auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlob(1, s); - stmt->BindText(2, s); + stmt.BindBlob(1, s); + stmt.BindText(2, s); s.at(0) = 'y'; - TF_ASSERT_OK(stmt->StepAndReset()); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ("hello", stmt->ColumnString(0)); - EXPECT_EQ("hello", stmt->ColumnString(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ("hello", stmt.ColumnString(0)); + EXPECT_EQ("hello", stmt.ColumnString(1)); } TEST_F(SqliteTest, UnsafeBind) { string s = "hello"; auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindBlobUnsafe(1, s); - stmt->BindTextUnsafe(2, s); + stmt.BindBlobUnsafe(1, s); + stmt.BindTextUnsafe(2, s); s.at(0) = 'y'; - TF_ASSERT_OK(stmt->StepAndReset()); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT a, b FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ("yello", stmt->ColumnString(0)); - EXPECT_EQ("yello", stmt->ColumnString(1)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ("yello", stmt.ColumnString(0)); + EXPECT_EQ("yello", stmt.ColumnString(1)); } TEST_F(SqliteTest, UnsafeColumn) { auto stmt = db_->Prepare("INSERT INTO T (a, b) VALUES (?, ?)"); - stmt->BindInt(1, 1); - stmt->BindText(2, "hello"); - TF_ASSERT_OK(stmt->StepAndReset()); - stmt->BindInt(1, 2); - stmt->BindText(2, "there"); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindInt(1, 1); + stmt.BindText(2, "hello"); + TF_ASSERT_OK(stmt.StepAndReset()); + stmt.BindInt(1, 2); + stmt.BindText(2, "there"); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT b FROM T ORDER BY a"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - const char* p = stmt->ColumnStringUnsafe(0); + TF_ASSERT_OK(stmt.Step(&is_done_)); + const char* p = stmt.ColumnStringUnsafe(0); EXPECT_EQ('h', *p); - TF_ASSERT_OK(stmt->Step(&is_done_)); + TF_ASSERT_OK(stmt.Step(&is_done_)); // This will actually happen, but it's not safe to test this behavior. // EXPECT_EQ('t', *p); } TEST_F(SqliteTest, NamedParameterBind) { auto stmt = db_->Prepare("INSERT INTO T (a) VALUES (:a)"); - stmt->BindText(":a", "lol"); - TF_ASSERT_OK(stmt->StepAndReset()); + stmt.BindText(":a", "lol"); + TF_ASSERT_OK(stmt.StepAndReset()); stmt = db_->Prepare("SELECT COUNT(*) FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); - EXPECT_EQ(1, stmt->ColumnInt(0)); + TF_ASSERT_OK(stmt.Step(&is_done_)); + EXPECT_EQ(1, stmt.ColumnInt(0)); stmt = db_->Prepare("SELECT a FROM T"); - TF_ASSERT_OK(stmt->Step(&is_done_)); + TF_ASSERT_OK(stmt.Step(&is_done_)); EXPECT_FALSE(is_done_); - EXPECT_EQ("lol", stmt->ColumnString(0)); + EXPECT_EQ("lol", stmt.ColumnString(0)); +} + +TEST_F(SqliteTest, Statement_DefaultConstructor) { + SqliteStatement stmt; + EXPECT_FALSE(stmt); + EXPECT_FALSE(stmt.StepAndReset().ok()); + stmt = db_->Prepare("INSERT INTO T (a) VALUES (1)"); + EXPECT_TRUE(stmt); + EXPECT_TRUE(stmt.StepAndReset().ok()); +} + +TEST_F(SqliteTest, Statement_MoveConstructor) { + SqliteStatement stmt{db_->Prepare("INSERT INTO T (a) VALUES (1)")}; + EXPECT_TRUE(stmt.StepAndReset().ok()); +} + +TEST_F(SqliteTest, Statement_MoveAssignment) { + SqliteStatement stmt1 = db_->Prepare("INSERT INTO T (a) VALUES (1)"); + SqliteStatement stmt2; + EXPECT_TRUE(stmt1.StepAndReset().ok()); + EXPECT_FALSE(stmt2.StepAndReset().ok()); + stmt2 = std::move(stmt1); + EXPECT_TRUE(stmt2.StepAndReset().ok()); +} + +TEST_F(SqliteTest, PrepareFailed) { + SqliteStatement s = db_->Prepare("SELECT"); + EXPECT_FALSE(s.status().ok()); + EXPECT_NE(string::npos, s.status().error_message().find("SELECT")); +} + +TEST_F(SqliteTest, BindFailed) { + SqliteStatement s = db_->Prepare("INSERT INTO T (a) VALUES (123)"); + EXPECT_TRUE(s.status().ok()); + EXPECT_EQ("", s.status().error_message()); + s.BindInt(1, 123); + EXPECT_FALSE(s.status().ok()); + EXPECT_NE(string::npos, + s.status().error_message().find("INSERT INTO T (a) VALUES (123)")); } } // namespace -} // namespace db } // namespace tensorflow diff --git a/tensorflow/core/lib/gtl/optional.cc b/tensorflow/core/lib/gtl/optional.cc new file mode 100644 index 0000000000000000000000000000000000000000..8dea073788a1ecaab023d149e0cdaf1ece9d49de --- /dev/null +++ b/tensorflow/core/lib/gtl/optional.cc @@ -0,0 +1,25 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/lib/gtl/optional.h" + +namespace tensorflow { +namespace gtl { + +nullopt_t::init_t nullopt_t::init; +extern const nullopt_t nullopt{nullopt_t::init}; + +} // namespace gtl +} // namespace tensorflow diff --git a/tensorflow/core/lib/io/inputbuffer.cc b/tensorflow/core/lib/io/inputbuffer.cc index 7efe2dc54341ee9780b6a0f3bd98e896f37a4700..4d35af49b2c719e3b3d5e434ef6a97badd9a9625 100644 --- a/tensorflow/core/lib/io/inputbuffer.cc +++ b/tensorflow/core/lib/io/inputbuffer.cc @@ -116,17 +116,35 @@ Status InputBuffer::ReadNBytes(int64 bytes_to_read, char* result, } Status InputBuffer::ReadVarint32Fallback(uint32* result) { + Status s = ReadVarintFallback(result, core::kMaxVarint32Bytes); + if (errors::IsDataLoss(s)) { + return errors::DataLoss("Stored data is too large to be a varint32."); + } + return s; +} + +Status InputBuffer::ReadVarint64Fallback(uint64* result) { + Status s = ReadVarintFallback(result, core::kMaxVarint64Bytes); + if (errors::IsDataLoss(s)) { + return errors::DataLoss("Stored data is too large to be a varint64."); + } + return s; +} + +template +Status InputBuffer::ReadVarintFallback(T* result, int max_bytes) { uint8 scratch = 0; - char* p = reinterpret_cast(&scratch); + auto* p = reinterpret_cast(&scratch); size_t unused_bytes_read = 0; *result = 0; - for (int shift = 0; shift <= 28; shift += 7) { + for (int index = 0; index < max_bytes; index++) { + int shift = 7 * index; TF_RETURN_IF_ERROR(ReadNBytes(1, p, &unused_bytes_read)); - *result |= (scratch & 127) << shift; + *result |= (static_cast(scratch) & 127) << shift; if (!(scratch & 128)) return Status::OK(); } - return errors::DataLoss("Stored data is too large to be a varint32."); + return errors::DataLoss("Stored data longer than ", max_bytes, " bytes."); } Status InputBuffer::SkipNBytes(int64 bytes_to_skip) { diff --git a/tensorflow/core/lib/io/inputbuffer.h b/tensorflow/core/lib/io/inputbuffer.h index 94a8cfd39be491a1485ddf12a861398e106c73bd..b3740f396ceb79fcdfd963100d9d53db892c5973 100644 --- a/tensorflow/core/lib/io/inputbuffer.h +++ b/tensorflow/core/lib/io/inputbuffer.h @@ -60,6 +60,9 @@ class InputBuffer { // Reads a single varint32. Status ReadVarint32(uint32* result); + // Reads a single varint64. + Status ReadVarint64(uint64* result); + // Like ReadNBytes() without returning the bytes read. Status SkipNBytes(int64 bytes_to_skip); @@ -82,6 +85,15 @@ class InputBuffer { // Internal slow-path routine used by ReadVarint32(). Status ReadVarint32Fallback(uint32* result); + // Internal slow-path routine used by ReadVarint64(). + Status ReadVarint64Fallback(uint64* result); + + // Helper method for reading a varint which can span at max `max_bytes`. + // If the varint is longer, a DataLoss error status is returned. + // If end of file is reached while reading, OutOfRange error is returned. + template + Status ReadVarintFallback(T* result, int max_bytes); + RandomAccessFile* file_; // Not owned int64 file_pos_; // Next position to read from in "file_" size_t size_; // Size of "buf_" @@ -109,6 +121,20 @@ inline Status InputBuffer::ReadVarint32(uint32* result) { } } +// Inlined for performance. +inline Status InputBuffer::ReadVarint64(uint64* result) { + if (pos_ + core::kMaxVarint64Bytes <= limit_) { + // Fast path: directly parse from buffered data. + // Reads strictly from the range [pos_, limit_). + const char* offset = core::GetVarint64Ptr(pos_, limit_, result); + if (offset == nullptr) return errors::OutOfRange("Parsed past limit."); + pos_ = const_cast(offset); + return Status::OK(); + } else { + return ReadVarint64Fallback(result); + } +} + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/inputbuffer_test.cc b/tensorflow/core/lib/io/inputbuffer_test.cc index 6771697a165c621f21342d2f396d5bc33e70db1d..6be1f819c2081dd4cc73853276d1cd94399614ff 100644 --- a/tensorflow/core/lib/io/inputbuffer_test.cc +++ b/tensorflow/core/lib/io/inputbuffer_test.cc @@ -329,5 +329,44 @@ TEST(InputBuffer, ReadVarint32) { } } +TEST(InputBuffer, ReadVarint64) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/inputbuffer_test"; + + // Generates data. + std::vector data; + uint64 i = 0; + for (; i < (1U << 10); i += 1) data.push_back(i); + for (; i < (1U << 15); i += 5) data.push_back(i); + for (; i < (1U << 31); i += 164817) data.push_back(i); + for (; i < (1ULL << 63); i += 16481797854795663UL) data.push_back(i); + data.push_back(std::numeric_limits::max()); + + // Writes the varints. + { + std::unique_ptr file; + TF_CHECK_OK(env->NewWritableFile(fname, &file)); + string varint; + for (uint64 number : data) { + varint.clear(); + core::PutVarint64(&varint, number); + TF_CHECK_OK(file->Append(StringPiece(varint))); + } + } + + for (auto buf_size : BufferSizes()) { + std::unique_ptr file; + TF_CHECK_OK(env->NewRandomAccessFile(fname, &file)); + io::InputBuffer in(file.get(), buf_size); + uint64 result = 0; + + for (uint64 expected : data) { + TF_ASSERT_OK(in.ReadVarint64(&result)); + EXPECT_EQ(expected, result); + } + EXPECT_TRUE(errors::IsOutOfRange(in.ReadVarint64(&result))); + } +} + } // namespace } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_buffers_test.cc b/tensorflow/core/lib/io/zlib_buffers_test.cc index 66ee68a916106a5fd0ecab1051c1ca982521941e..156c712db87d6c248ec880af52a4edef1429236d 100644 --- a/tensorflow/core/lib/io/zlib_buffers_test.cc +++ b/tensorflow/core/lib/io/zlib_buffers_test.cc @@ -68,25 +68,25 @@ void TestAllCombinations(CompressionOptions input_options, for (auto input_buf_size : InputBufferSizes()) { for (auto output_buf_size : OutputBufferSizes()) { std::unique_ptr file_writer; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); string result; ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); - TF_CHECK_OK(out.Append(StringPiece(data))); - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, input_options); - TF_EXPECT_OK(in.ReadNBytes(data.size(), &result)); + TF_ASSERT_OK(in.ReadNBytes(data.size(), &result)); EXPECT_EQ(result, data); } } @@ -118,24 +118,24 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size, string actual_result; string expected_result; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); for (int i = 0; i < num_writes; i++) { - TF_CHECK_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Append(StringPiece(data))); if (with_flush) { - TF_CHECK_OK(out.Flush()); + TF_ASSERT_OK(out.Flush()); } strings::StrAppend(&expected_result, data); } - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, @@ -143,7 +143,7 @@ void TestMultipleWrites(uint8 input_buf_size, uint8 output_buf_size, for (int i = 0; i < num_writes; i++) { string decompressed_output; - TF_EXPECT_OK(in.ReadNBytes(data.size(), &decompressed_output)); + TF_ASSERT_OK(in.ReadNBytes(data.size(), &decompressed_output)); strings::StrAppend(&actual_result, decompressed_output); } @@ -170,19 +170,19 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) { string data = GenTestString(10); std::unique_ptr file_writer; - TF_CHECK_OK(env->NewWritableFile(fname, &file_writer)); + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); string result; ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, output_options); - TF_CHECK_OK(out.Init()); + TF_ASSERT_OK(out.Init()); - TF_CHECK_OK(out.Append(StringPiece(data))); - TF_CHECK_OK(out.Close()); - TF_CHECK_OK(file_writer->Flush()); - TF_CHECK_OK(file_writer->Close()); + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); std::unique_ptr file_reader; - TF_CHECK_OK(env->NewRandomAccessFile(fname, &file_reader)); + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); std::unique_ptr input_stream( new RandomAccessInputStream(file_reader.get())); ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, @@ -192,5 +192,129 @@ TEST(ZlibInputStream, FailsToReadIfWindowBitsAreIncompatible) { CHECK(read_status.error_message().find("inflate() failed") != string::npos); } +void WriteCompressedFile(Env* env, const string& fname, int input_buf_size, + int output_buf_size, + const CompressionOptions& output_options, + const string& data) { + std::unique_ptr file_writer; + TF_ASSERT_OK(env->NewWritableFile(fname, &file_writer)); + + ZlibOutputBuffer out(file_writer.get(), input_buf_size, output_buf_size, + output_options); + TF_ASSERT_OK(out.Init()); + + TF_ASSERT_OK(out.Append(StringPiece(data))); + TF_ASSERT_OK(out.Close()); + TF_ASSERT_OK(file_writer->Flush()); + TF_ASSERT_OK(file_writer->Close()); +} + +void TestTell(CompressionOptions input_options, + CompressionOptions output_options) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/zlib_buffers_test"; + for (auto file_size : NumCopies()) { + string data = GenTestString(file_size); + for (auto input_buf_size : InputBufferSizes()) { + for (auto output_buf_size : OutputBufferSizes()) { + // Write the compressed file. + WriteCompressedFile(env, fname, input_buf_size, output_buf_size, + output_options, data); + + // Boiler-plate to set up ZlibInputStream. + std::unique_ptr file_reader; + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); + std::unique_ptr input_stream( + new RandomAccessInputStream(file_reader.get())); + ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, + input_options); + + string first_half(data, 0, data.size() / 2); + string bytes_read; + + // Read the first half of the uncompressed file and expect that Tell() + // returns half the uncompressed length of the file. + TF_ASSERT_OK(in.ReadNBytes(first_half.size(), &bytes_read)); + EXPECT_EQ(in.Tell(), first_half.size()); + EXPECT_EQ(bytes_read, first_half); + + // Read the remaining half of the uncompressed file and expect that + // Tell() points past the end of file. + string second_half; + TF_ASSERT_OK( + in.ReadNBytes(data.size() - first_half.size(), &second_half)); + EXPECT_EQ(in.Tell(), data.size()); + bytes_read.append(second_half); + + // Expect that the file is correctly read. + EXPECT_EQ(bytes_read, data); + } + } + } +} + +void TestSkipNBytes(CompressionOptions input_options, + CompressionOptions output_options) { + Env* env = Env::Default(); + string fname = testing::TmpDir() + "/zlib_buffers_test"; + for (auto file_size : NumCopies()) { + string data = GenTestString(file_size); + for (auto input_buf_size : InputBufferSizes()) { + for (auto output_buf_size : OutputBufferSizes()) { + // Write the compressed file. + WriteCompressedFile(env, fname, input_buf_size, output_buf_size, + output_options, data); + + // Boiler-plate to set up ZlibInputStream. + std::unique_ptr file_reader; + TF_ASSERT_OK(env->NewRandomAccessFile(fname, &file_reader)); + std::unique_ptr input_stream( + new RandomAccessInputStream(file_reader.get())); + ZlibInputStream in(input_stream.get(), input_buf_size, output_buf_size, + input_options); + + size_t data_half_size = data.size() / 2; + string second_half(data, data_half_size, data.size() - data_half_size); + + // Skip past the first half of the file and expect Tell() returns + // correctly. + TF_ASSERT_OK(in.SkipNBytes(data_half_size)); + EXPECT_EQ(in.Tell(), data_half_size); + + // Expect that second half is read correctly and Tell() returns past + // end of file after reading complete file. + string bytes_read; + TF_ASSERT_OK(in.ReadNBytes(second_half.size(), &bytes_read)); + EXPECT_EQ(bytes_read, second_half); + EXPECT_EQ(in.Tell(), data.size()); + } + } + } +} + +TEST(ZlibInputStream, TellDefaultOptions) { + TestTell(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT()); +} + +TEST(ZlibInputStream, TellRawDeflate) { + TestTell(CompressionOptions::RAW(), CompressionOptions::RAW()); +} + +TEST(ZlibInputStream, TellGzip) { + TestTell(CompressionOptions::GZIP(), CompressionOptions::GZIP()); +} + +TEST(ZlibInputStream, SkipNBytesDefaultOptions) { + TestSkipNBytes(CompressionOptions::DEFAULT(), CompressionOptions::DEFAULT()); +} + +TEST(ZlibInputStream, SkipNBytesRawDeflate) { + TestSkipNBytes(CompressionOptions::RAW(), CompressionOptions::RAW()); +} + +TEST(ZlibInputStream, SkipNBytesGzip) { + TestSkipNBytes(CompressionOptions::GZIP(), CompressionOptions::GZIP()); +} + } // namespace io } // namespace tensorflow diff --git a/tensorflow/core/lib/io/zlib_inputstream.cc b/tensorflow/core/lib/io/zlib_inputstream.cc index 4999d5cc90bb44e368b2489c3afd13188a494d51..984fbc2810c28ac818ab8f5a86451bed4e101605 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.cc +++ b/tensorflow/core/lib/io/zlib_inputstream.cc @@ -32,7 +32,8 @@ ZlibInputStream::ZlibInputStream( z_stream_input_(new Bytef[input_buffer_capacity_]), z_stream_output_(new Bytef[output_buffer_capacity_]), zlib_options_(zlib_options), - z_stream_(new z_stream) { + z_stream_(new z_stream), + bytes_read_(0) { InitZlibBuffer(); } @@ -45,6 +46,7 @@ ZlibInputStream::~ZlibInputStream() { Status ZlibInputStream::Reset() { TF_RETURN_IF_ERROR(input_stream_->Reset()); InitZlibBuffer(); + bytes_read_ = 0; return Status::OK(); } @@ -127,6 +129,7 @@ size_t ZlibInputStream::ReadBytesFromCache(size_t bytes_to_read, result->append(next_unread_byte_, can_read_bytes); next_unread_byte_ += can_read_bytes; } + bytes_read_ += can_read_bytes; return can_read_bytes; } @@ -170,8 +173,7 @@ Status ZlibInputStream::ReadNBytes(int64 bytes_to_read, string* result) { return Status::OK(); } -// TODO(srbs): Implement this. -int64 ZlibInputStream::Tell() const { return -1; } +int64 ZlibInputStream::Tell() const { return bytes_read_; } Status ZlibInputStream::Inflate() { int error = inflate(z_stream_.get(), zlib_options_.flush_mode); diff --git a/tensorflow/core/lib/io/zlib_inputstream.h b/tensorflow/core/lib/io/zlib_inputstream.h index 8faa7dcb8f4139746132934813602bcb4a4e0ea9..9c7e14441ce92756f2bc0716210dc41652c9e105 100644 --- a/tensorflow/core/lib/io/zlib_inputstream.h +++ b/tensorflow/core/lib/io/zlib_inputstream.h @@ -132,6 +132,9 @@ class ZlibInputStream : public InputStreamInterface { // Returns the size of [next_unread_byte_, z_stream_->next_out) size_t NumUnreadBytes() const; + // Number of *uncompressed* bytes that have been read from this stream. + int64 bytes_read_; + TF_DISALLOW_COPY_AND_ASSIGN(ZlibInputStream); }; diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 15b09c2c169e707a42292a36b30bb61f375505cd..cdf370399c0cf892f5003bddc99ae2ac259cad22 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -739,7 +739,7 @@ REGISTER_OP("Diag") .Attr("T: {float, double, int32, int64, complex64, complex128}") .SetShapeFn([](InferenceContext* c) { ShapeHandle in = c->input(0); - TF_RETURN_IF_ERROR(c->WithRankAtMost(in, 3, &in)); + TF_RETURN_IF_ERROR(c->WithRankAtLeast(in, 1, &in)); // Output shape is original concatenated with itself. ShapeHandle out; TF_RETURN_IF_ERROR(c->Concatenate(in, in, &out)); @@ -767,7 +767,7 @@ tf.diag(diagonal) ==> [[1, 0, 0, 0] [0, 0, 0, 4]] ``` -diagonal: Rank k tensor where k is at most 3. +diagonal: Rank k tensor where k is at most 1. )doc"); // -------------------------------------------------------------------------- @@ -783,9 +783,9 @@ REGISTER_OP("DiagPart") } // Rank must be even, and result will have rank . const int32 rank = c->Rank(in); - if ((rank % 2) != 0 || rank > 6) { + if ((rank % 2) != 0 || rank <= 0) { return errors::InvalidArgument( - "Input must have even rank <= 6, input rank is ", rank); + "Input must have even and non-zero rank, input rank is ", rank); } const int32 mid = rank / 2; @@ -820,7 +820,7 @@ For example: tf.diag_part(input) ==> [1, 2, 3, 4] ``` -input: Rank k tensor where k is 2, 4, or 6. +input: Rank k tensor where k is even and not zero. diagonal: The extracted diagonal. )doc"); @@ -4859,6 +4859,9 @@ REGISTER_OP("QuantizeV2") .Output("output_max: float") .Attr("T: quantizedtype") .Attr("mode: {'MIN_COMBINED', 'MIN_FIRST', 'SCALED'} = 'MIN_COMBINED'") + .Attr( + "round_mode: {'HALF_AWAY_FROM_ZERO', 'HALF_TO_EVEN'} = " + "'HALF_AWAY_FROM_ZERO'") .SetShapeFn([](InferenceContext* c) { TF_RETURN_IF_ERROR(shape_inference::UnchangedShape(c)); ShapeHandle unused; @@ -4873,7 +4876,9 @@ Quantize the 'input' tensor of type float to 'output' tensor of type 'T'. [min_range, max_range] are scalar floats that specify the range for the 'input' data. The 'mode' attribute controls exactly which calculations are -used to convert the float values to their quantized equivalents. +used to convert the float values to their quantized equivalents. The +'round_mode' attribute controls which rounding tie-breaking algorithm is used +when rounding float values to their quantized equivalents. In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: @@ -4897,10 +4902,10 @@ with the range of qint8. If the mode is 'MIN_FIRST', then this approach is used: ``` -number_of_steps = 1 << (# of bits in T) -range_adjust = number_of_steps / (number_of_steps - 1) +num_discrete_values = 1 << (# of bits in T) +range_adjust = num_discrete_values / (num_discrete_values - 1) range = (range_max - range_min) * range_adjust -range_scale = number_of_steps / range +range_scale = num_discrete_values / range quantized = round(input * range_scale) - round(range_min * range_scale) + numeric_limits::min() quantized = max(quantized, numeric_limits::min()) @@ -4950,7 +4955,7 @@ From this we compute our scaling factor, s: Now we can quantize the elements of our tensor: ```c++ -result = (input * s).round_to_nearest() +result = round(input * s) ``` One thing to watch out for is that the operator may choose to adjust the @@ -5012,10 +5017,10 @@ each value by 128 prior to casting. If the mode is 'MIN_FIRST', then this approach is used: ```c++ -number_of_steps = 1 << (# of bits in T) -range_adjust = number_of_steps / (number_of_steps - 1) +num_discrete_values = 1 << (# of bits in T) +range_adjust = num_discrete_values / (num_discrete_values - 1) range = (range_max - range_min) * range_adjust -range_scale = range / number_of_steps +range_scale = range / num_discrete_values const double offset_input = static_cast(input) - lowest_quantized; result = range_min + ((input - numeric_limits::min()) * range_scale) ``` diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index a5d7a32e05f4688a3a7a7a21eaa9b18d44a21b15..94eb120175555d8d51b9be1ff98676a9dc4fff07 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -186,21 +186,20 @@ TEST(ArrayOpsTest, Identity_ShapeFnHandles) { TEST(ArrayOpsTest, Diag_ShapeFn) { ShapeInferenceTestOp op("Diag"); INFER_OK(op, "?", "?"); - INFER_OK(op, "[]", "[]"); INFER_OK(op, "[1,?,3]", "[d0_0,d0_1,d0_2,d0_0,d0_1,d0_2]"); - INFER_ERROR("Shape must be at most rank 3 but is rank 4", op, "[?,1,2,3]"); + INFER_OK(op, "[?,1,2,3]", "[d0_0,d0_1,d0_2,d0_3,d0_0,d0_1,d0_2,d0_3]"); + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[]"); } TEST(ArrayOpsTest, DiagPart_ShapeFn) { ShapeInferenceTestOp op("DiagPart"); INFER_OK(op, "?", "?"); - INFER_OK(op, "[]", "[]"); INFER_OK(op, "[1,?,?,4]", "[d0_0,d0_3]"); INFER_OK(op, "[1,?,3,?,4,3]", "[d0_0,d0_4,d0_2|d0_5]"); - INFER_ERROR("Input must have even rank <= 6, input rank is 1", op, "[?]"); - INFER_ERROR("Input must have even rank <= 6, input rank is 3", op, "[1,2,3]"); - INFER_ERROR("Input must have even rank <= 6, input rank is 8", op, - "[1,2,3,?,?,?,?,?]"); + INFER_OK(op, "[1,2,3,?,?,?,?,4]", "[d0_0,d0_1,d0_2,d0_7]"); + INFER_ERROR("Input must have even and non-zero rank", op, "[]"); + INFER_ERROR("Input must have even and non-zero rank", op, "[?]"); + INFER_ERROR("Input must have even and non-zero rank", op, "[1,2,3]"); INFER_ERROR("Dimensions must be equal, but are 2 and 10", op, "[1,2,?,10]"); } diff --git a/tensorflow/core/ops/bitwise_ops.cc b/tensorflow/core/ops/bitwise_ops.cc index 3ffc4ab74af71abf515c838d73fe9fe0b8863070..2889953bdbc614bc4e56245e45c08d913cfd5255 100644 --- a/tensorflow/core/ops/bitwise_ops.cc +++ b/tensorflow/core/ops/bitwise_ops.cc @@ -23,7 +23,7 @@ namespace tensorflow { REGISTER_OP("Invert") .Input("x: T") .Output("y: T") - .Attr("T: {int8, int16, int32, int64, uint8, uint16}") + .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Flips all bits elementwise. @@ -32,18 +32,18 @@ The result will have exactly those bits set, that are not set in `x`. The computation is performed on the underlying representation of x. )doc"); -#define BINARY_BITWISE() \ - Input("x: T") \ - .Input("y: T") \ - .Output("z: T") \ - .SetIsCommutative() \ - .Attr("T: {int8, int16, int32, int64, uint8, uint16}") \ +#define BINARY_BITWISE() \ + Input("x: T") \ + .Input("y: T") \ + .Output("z: T") \ + .SetIsCommutative() \ + .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") \ .SetShapeFn(shape_inference::UnchangedShape) REGISTER_OP("PopulationCount") .Input("x: T") .Output("y: uint8") - .Attr("T: {int8, int16, int32, int64, uint8, uint16}") + .Attr("T: {int8, int16, int32, int64, uint8, uint16, uint32, uint64}") .SetShapeFn(shape_inference::UnchangedShape) .Doc(R"doc( Computes element-wise population count (a.k.a. popcount, bitsum, bitcount). @@ -56,25 +56,52 @@ representation of that entry. 8- or 16-bit inputs and then aggregate the resulting counts. )doc"); -REGISTER_OP("BitwiseAnd").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("BitwiseAnd") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise AND of `x` and `y`. The result will have those bits set, that are set in both `x` and `y`. The computation is performed on the underlying representations of `x` and `y`. )doc"); -REGISTER_OP("BitwiseOr").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("BitwiseOr") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise OR of `x` and `y`. The result will have those bits set, that are set in `x`, `y` or both. The computation is performed on the underlying representations of `x` and `y`. )doc"); -REGISTER_OP("BitwiseXor").BINARY_BITWISE().Doc(R"doc( +REGISTER_OP("BitwiseXor") + .BINARY_BITWISE() + .Doc(R"doc( Elementwise computes the bitwise XOR of `x` and `y`. The result will have those bits set, that are different in `x` and `y`. The computation is performed on the underlying representations of `x` and `y`. )doc"); +REGISTER_OP("LeftShift") + .BINARY_BITWISE() + .Doc(R"doc( +Elementwise computes the bitwise left-shift of `x` and `y`. + +If `y` is negative, or greater than or equal to the width of `x` in bits the +result is implementation defined. +)doc"); + +REGISTER_OP("RightShift") + .BINARY_BITWISE() + .Doc(R"doc( +Elementwise computes the bitwise right-shift of `x` and `y`. + +Performs a logical shift for unsigned integer types, and an arithmetic shift +for signed integer types. + +If `y` is negative, or greater than or equal to than the width of `x` in bits +the result is implementation defined. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/core/ops/compat/backwards_compatibility_test.cc b/tensorflow/core/ops/compat/backwards_compatibility_test.cc index 6e05ae4be4fb967ac8dcc5a03fa548c7cb6c0f9b..add05d6610ae62158b653d27699f61bc511ee3b6 100644 --- a/tensorflow/core/ops/compat/backwards_compatibility_test.cc +++ b/tensorflow/core/ops/compat/backwards_compatibility_test.cc @@ -25,9 +25,8 @@ namespace tensorflow { namespace { TEST(BackwardsCompatibilityTest, IsCompatible) { - OpCompatibilityLib compatibility("tensorflow/core/ops", - strings::StrCat("v", TF_MAJOR_VERSION), - nullptr); + OpCompatibilityLib compatibility( + "tensorflow/core/ops", strings::StrCat("v", TF_MAJOR_VERSION), nullptr); Env* env = Env::Default(); int changed_ops = 0; diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt index d93a4ff933bb625a897672170d3697054113a497..f385ef54f1c3a3ca22f49427b75e55c9e240936b 100644 --- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt +++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt @@ -39,6 +39,54 @@ op { } } } +op { + name: "AccumulateNV2" + input_arg { + name: "inputs" + type_attr: "T" + number_attr: "N" + } + output_arg { + name: "sum" + type_attr: "T" + } + attr { + name: "N" + type: "int" + has_minimum: true + minimum: 1 + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "shape" + type: "shape" + } + is_aggregate: true + is_commutative: true +} op { name: "AccumulatorApplyGradient" input_arg { @@ -509,6 +557,41 @@ op { } is_stateful: true } +op { + name: "AddV2" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + is_aggregate: true + is_commutative: true +} op { name: "AdjustContrast" input_arg { @@ -5733,6 +5816,68 @@ op { } is_commutative: true } +op { + name: "BitwiseAnd" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} +op { + name: "BitwiseOr" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + } + } + } + is_commutative: true +} op { name: "BitwiseOr" input_arg { @@ -5747,6 +5892,38 @@ op { name: "z" type_attr: "T" } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} +op { + name: "BitwiseXor" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } attr { name: "T" type: "type" @@ -5788,6 +5965,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -7558,11 +7737,7 @@ op { } } op { - name: "CropAndResizeGradBoxes" - input_arg { - name: "grads" - type: DT_FLOAT - } + name: "CropAndResize" input_arg { name: "image" type_attr: "T" @@ -7575,8 +7750,12 @@ op { name: "box_ind" type: DT_INT32 } + input_arg { + name: "crop_size" + type: DT_INT32 + } output_arg { - name: "output" + name: "crops" type: DT_FLOAT } attr { @@ -7585,6 +7764,7 @@ op { allowed_values { list { type: DT_UINT8 + type: DT_UINT16 type: DT_INT8 type: DT_INT16 type: DT_INT32 @@ -7607,13 +7787,24 @@ op { } } } + attr { + name: "extrapolation_value" + type: "float" + default_value { + f: 0 + } + } } op { - name: "CropAndResizeGradImage" + name: "CropAndResizeGradBoxes" input_arg { name: "grads" type: DT_FLOAT } + input_arg { + name: "image" + type_attr: "T" + } input_arg { name: "boxes" type: DT_FLOAT @@ -7622,21 +7813,22 @@ op { name: "box_ind" type: DT_INT32 } - input_arg { - name: "image_size" - type: DT_INT32 - } output_arg { name: "output" - type_attr: "T" + type: DT_FLOAT } attr { name: "T" type: "type" allowed_values { list { - type: DT_FLOAT + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 type: DT_HALF + type: DT_FLOAT type: DT_DOUBLE } } @@ -7655,44 +7847,142 @@ op { } } op { - name: "Cross" + name: "CropAndResizeGradBoxes" input_arg { - name: "a" - type_attr: "T" + name: "grads" + type: DT_FLOAT } input_arg { - name: "b" + name: "image" type_attr: "T" } + input_arg { + name: "boxes" + type: DT_FLOAT + } + input_arg { + name: "box_ind" + type: DT_INT32 + } output_arg { - name: "product" - type_attr: "T" + name: "output" + type: DT_FLOAT } attr { name: "T" type: "type" allowed_values { list { - type: DT_FLOAT - type: DT_DOUBLE - type: DT_INT32 - type: DT_INT64 type: DT_UINT8 - type: DT_INT16 - type: DT_INT8 type: DT_UINT16 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE } } } -} -op { - name: "Cross" - input_arg { - name: "a" - type_attr: "T" - } - input_arg { + attr { + name: "method" + type: "string" + default_value { + s: "bilinear" + } + allowed_values { + list { + s: "bilinear" + } + } + } +} +op { + name: "CropAndResizeGradImage" + input_arg { + name: "grads" + type: DT_FLOAT + } + input_arg { + name: "boxes" + type: DT_FLOAT + } + input_arg { + name: "box_ind" + type: DT_INT32 + } + input_arg { + name: "image_size" + type: DT_INT32 + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_HALF + type: DT_DOUBLE + } + } + } + attr { + name: "method" + type: "string" + default_value { + s: "bilinear" + } + allowed_values { + list { + s: "bilinear" + } + } + } +} +op { + name: "Cross" + input_arg { + name: "a" + type_attr: "T" + } + input_arg { + name: "b" + type_attr: "T" + } + output_arg { + name: "product" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + } + } + } +} +op { + name: "Cross" + input_arg { + name: "a" + type_attr: "T" + } + input_arg { name: "b" type_attr: "T" } @@ -9486,6 +9776,18 @@ op { } } } +op { + name: "DeserializeIterator" + input_arg { + name: "resource_handle" + type: DT_RESOURCE + } + input_arg { + name: "serialized" + type: DT_VARIANT + } + is_stateful: true +} op { name: "DeserializeManySparse" input_arg { @@ -13425,6 +13727,50 @@ op { } is_stateful: true } +op { + name: "HistogramFixedWidth" + input_arg { + name: "values" + type_attr: "T" + } + input_arg { + name: "value_range" + type_attr: "T" + } + input_arg { + name: "nbins" + type: DT_INT32 + } + output_arg { + name: "out" + type_attr: "dtype" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "dtype" + type: "type" + default_value { + type: DT_INT32 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } +} op { name: "HistogramSummary" input_arg { @@ -14315,6 +14661,33 @@ op { } } } +op { + name: "Invert" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } +} op { name: "InvertPermutation" input_arg { @@ -14818,6 +15191,38 @@ op { } is_stateful: true } +op { + name: "LeftShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} op { name: "Less" input_arg { @@ -19543,6 +19948,47 @@ op { } is_commutative: true } +op { + name: "NthElement" + input_arg { + name: "input" + type_attr: "T" + } + input_arg { + name: "n" + type: DT_INT32 + } + output_arg { + name: "values" + type_attr: "T" + } + attr { + name: "reverse" + type: "bool" + default_value { + b: false + } + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } +} op { name: "OneHot" input_arg { @@ -20313,7 +20759,7 @@ op { } } op { - name: "ParallelMapDataset" + name: "ParallelInterleaveDataset" input_arg { name: "input_dataset" type: DT_VARIANT @@ -20323,12 +20769,60 @@ op { type_list_attr: "Targuments" } input_arg { - name: "num_parallel_calls" - type: DT_INT32 + name: "cycle_length" + type: DT_INT64 } - output_arg { - name: "handle" - type: DT_VARIANT + input_arg { + name: "block_length" + type: DT_INT64 + } + input_arg { + name: "sloppy" + type: DT_BOOL + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } +} +op { + name: "ParallelMapDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "num_parallel_calls" + type: DT_INT32 + } + output_arg { + name: "handle" + type: DT_VARIANT } attr { name: "f" @@ -20883,6 +21377,33 @@ op { } } } +op { + name: "PopulationCount" + input_arg { + name: "x" + type_attr: "T" + } + output_arg { + name: "y" + type: DT_UINT8 + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } +} op { name: "Pow" input_arg { @@ -21756,6 +22277,73 @@ op { } } } +op { + name: "QuantizeV2" + input_arg { + name: "input" + type: DT_FLOAT + } + input_arg { + name: "min_range" + type: DT_FLOAT + } + input_arg { + name: "max_range" + type: DT_FLOAT + } + output_arg { + name: "output" + type_attr: "T" + } + output_arg { + name: "output_min" + type: DT_FLOAT + } + output_arg { + name: "output_max" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT16 + type: DT_QUINT16 + type: DT_QINT32 + } + } + } + attr { + name: "mode" + type: "string" + default_value { + s: "MIN_COMBINED" + } + allowed_values { + list { + s: "MIN_COMBINED" + s: "MIN_FIRST" + s: "SCALED" + } + } + } + attr { + name: "round_mode" + type: "string" + default_value { + s: "HALF_AWAY_FROM_ZERO" + } + allowed_values { + list { + s: "HALF_AWAY_FROM_ZERO" + s: "HALF_TO_EVEN" + } + } + } +} op { name: "QuantizedAdd" input_arg { @@ -24996,6 +25584,45 @@ op { } } } +op { + name: "ResizeArea" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} op { name: "ResizeBicubic" input_arg { @@ -25034,6 +25661,45 @@ op { } } } +op { + name: "ResizeBicubic" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} op { name: "ResizeBicubicGrad" input_arg { @@ -25077,21 +25743,93 @@ op { type: DT_INT32 } output_arg { - name: "resized_images" - type: DT_FLOAT + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} +op { + name: "ResizeBilinear" + input_arg { + name: "images" + type_attr: "T" + } + input_arg { + name: "size" + type: DT_INT32 + } + output_arg { + name: "resized_images" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_UINT8 + type: DT_INT16 + type: DT_UINT16 + type: DT_INT32 + type: DT_INT64 + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "align_corners" + type: "bool" + default_value { + b: false + } + } +} +op { + name: "ResizeBilinearGrad" + input_arg { + name: "grads" + type: DT_FLOAT + } + input_arg { + name: "original_image" + type_attr: "T" + } + output_arg { + name: "output" + type_attr: "T" } attr { name: "T" type: "type" allowed_values { list { - type: DT_UINT8 - type: DT_INT8 - type: DT_INT16 - type: DT_INT32 - type: DT_INT64 - type: DT_HALF type: DT_FLOAT + type: DT_HALF type: DT_DOUBLE } } @@ -25105,17 +25843,17 @@ op { } } op { - name: "ResizeBilinearGrad" + name: "ResizeNearestNeighbor" input_arg { - name: "grads" - type: DT_FLOAT + name: "images" + type_attr: "T" } input_arg { - name: "original_image" - type_attr: "T" + name: "size" + type: DT_INT32 } output_arg { - name: "output" + name: "resized_images" type_attr: "T" } attr { @@ -25123,8 +25861,13 @@ op { type: "type" allowed_values { list { - type: DT_FLOAT + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 type: DT_HALF + type: DT_FLOAT type: DT_DOUBLE } } @@ -25156,9 +25899,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -26784,6 +27528,32 @@ op { } is_stateful: true } +op { + name: "ResourceCountUpTo" + input_arg { + name: "resource" + type: DT_RESOURCE + } + output_arg { + name: "output" + type_attr: "T" + } + attr { + name: "limit" + type: "int" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + is_stateful: true +} op { name: "ResourceGather" input_arg { @@ -26919,6 +27689,56 @@ op { } is_stateful: true } +op { + name: "ResourceScatterUpdate" + input_arg { + name: "resource" + type: DT_RESOURCE + } + input_arg { + name: "indices" + type_attr: "Tindices" + } + input_arg { + name: "updates" + type_attr: "dtype" + } + attr { + name: "dtype" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + is_stateful: true +} op { name: "ResourceSparseApplyAdadelta" input_arg { @@ -28571,18 +29391,6 @@ op { } is_stateful: true } -op { - name: "RestoreIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - is_stateful: true -} op { name: "RestoreSlice" input_arg { @@ -28983,6 +29791,38 @@ op { } } } +op { + name: "RightShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + is_commutative: true +} op { name: "Rint" input_arg { @@ -29334,18 +30174,6 @@ op { } is_stateful: true } -op { - name: "SaveIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - is_stateful: true -} op { name: "SaveSlices" input_arg { @@ -31275,6 +32103,18 @@ op { } } } +op { + name: "SerializeIterator" + input_arg { + name: "resource_handle" + type: DT_RESOURCE + } + output_arg { + name: "serialized" + type: DT_VARIANT + } + is_stateful: true +} op { name: "SerializeManySparse" input_arg { @@ -31922,95 +32762,6 @@ op { } } } -op { - name: "SloppyInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - is_stateful: true -} -op { - name: "SloppyInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } -} op { name: "Softmax" input_arg { diff --git a/tensorflow/core/ops/data_flow_ops.cc b/tensorflow/core/ops/data_flow_ops.cc index 8e24ea70cba8b86426d52048f1bd624c777a82fe..3b1ed217ce1b444b0601d5a1b1d599489ee33644 100644 --- a/tensorflow/core/ops/data_flow_ops.cc +++ b/tensorflow/core/ops/data_flow_ops.cc @@ -2225,7 +2225,6 @@ this op will block until it does. This Op is optimized for performance. )doc"); - REGISTER_OP("StageSize") .Output("size: int32") .Attr("capacity: int >= 0 = 0") @@ -2354,7 +2353,6 @@ REGISTER_OP("MapIncompleteSize") Op returns the number of incomplete elements in the underlying container. )doc"); - REGISTER_OP("MapClear") .Attr("capacity: int >= 0 = 0") .Attr("memory_limit: int >= 0 = 0") @@ -2367,7 +2365,6 @@ REGISTER_OP("MapClear") Op removes all elements in the underlying container. )doc"); - // OrderedMap REGISTER_OP("OrderedMapStage") .Input("key: int64") diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc index 566049179a1f424937c17409b182115f75621b59..f5122139645e2d3360bdcdbde29335ccaca79fbb 100644 --- a/tensorflow/core/ops/dataset_ops.cc +++ b/tensorflow/core/ops/dataset_ops.cc @@ -285,11 +285,12 @@ f: A function mapping elements of `input_dataset`, concatenated with `output_types` and `output_shapes`. )doc"); -REGISTER_OP("SloppyInterleaveDataset") +REGISTER_OP("ParallelInterleaveDataset") .Input("input_dataset: variant") .Input("other_arguments: Targuments") .Input("cycle_length: int64") .Input("block_length: int64") + .Input("sloppy: bool") .Output("handle: variant") .Attr("f: func") .Attr("Targuments: list(type) >= 0") @@ -598,24 +599,6 @@ This operation may be executed multiple times. Each execution will reset the iterator in `iterator` to the first element of `dataset`. )doc"); -REGISTER_OP("SaveIterator") - .Input("iterator: resource") - .Input("path: string") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Saves the state of the `iterator` at `path`. - -This state can be restored using "RestoreIterator". -)doc"); - -REGISTER_OP("RestoreIterator") - .Input("iterator: resource") - .Input("path: string") - .SetShapeFn(shape_inference::NoOutputs) - .Doc(R"doc( -Restores the state of the `iterator` from the checkpoint saved at `path` using "SaveIterator". -)doc"); - REGISTER_OP("OneShotIterator") .Output("handle: resource") .Attr("dataset_factory: func") @@ -737,4 +720,28 @@ output_shapes: If specified, defines the shape of each tuple component in an element produced by the resulting iterator. )doc"); +REGISTER_OP("SerializeIterator") + .Input("resource_handle: resource") + .Output("serialized: variant") + .SetShapeFn(shape_inference::ScalarShape) + .Doc(R"doc( +Converts the given `resource_handle` representing an iterator to a variant tensor. + +resource_handle: A handle to an iterator resource. +serialized: A variant tensor storing the state of the iterator contained in the + resource. +)doc"); + +REGISTER_OP("DeserializeIterator") + .Input("resource_handle: resource") + .Input("serialized: variant") + .SetShapeFn(shape_inference::NoOutputs) + .Doc(R"doc( +Converts the given variant tensor to an iterator and stores it in the given resource. + +resource_handle: A handle to an iterator resource. +serialized: A variant tensor storing the state of the iterator contained in the + resource. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc index 66765a33333d39d7f36d79ad97dfaaf67af9a1ac..c3f80064150ba0dcce1173de1d02142cf3dc6621 100644 --- a/tensorflow/core/ops/image_ops.cc +++ b/tensorflow/core/ops/image_ops.cc @@ -453,7 +453,36 @@ REGISTER_OP("DecodeAndCropJpeg") .Attr("acceptable_fraction: float = 1.0") .Attr("dct_method: string = ''") .Output("image: uint8") - .SetShapeFn(DecodeImageShapeFn) + .SetShapeFn([](InferenceContext* c) { + ShapeHandle unused; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused)); + DimensionHandle channels_dim = c->UnknownDim(); + DimensionHandle h = c->UnknownDim(); + DimensionHandle w = c->UnknownDim(); + + int32 channels; + TF_RETURN_IF_ERROR(c->GetAttr("channels", &channels)); + if (channels != 0) { + if (channels < 0) { + return errors::InvalidArgument("channels must be non-negative, got ", + channels); + } + channels_dim = c->MakeDim(channels); + } + + DimensionHandle unused_dim; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused)); + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(unused, 0), 4, &unused_dim)); + + const Tensor* crop_window = c->input_tensor(1); + if (crop_window != nullptr) { + auto crop_window_vec = crop_window->vec(); + h = c->MakeDim(crop_window_vec(2)); + w = c->MakeDim(crop_window_vec(3)); + } + c->set_output(0, c->MakeShape({h, w, channels_dim})); + return Status::OK(); + }) .Doc(strings::StrCat(R"doc( Decode and Crop a JPEG-encoded image to a uint8 tensor. )doc", @@ -896,27 +925,27 @@ use_image_if_no_bounding_boxes: Controls behavior if no bounding boxes supplied. )doc"); REGISTER_OP("SampleDistortedBoundingBoxV2") - .Input("image_size: T") - .Input("bounding_boxes: float") - .Input("min_object_covered: float") - .Output("begin: T") - .Output("size: T") - .Output("bboxes: float") - .Attr("T: {uint8, int8, int16, int32, int64}") - .Attr("seed: int = 0") - .Attr("seed2: int = 0") - .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]") - .Attr("area_range: list(float) = [0.05, 1.0]") - .Attr("max_attempts: int = 100") - .Attr("use_image_if_no_bounding_boxes: bool = false") - .SetIsStateful() - .SetShapeFn([](InferenceContext* c) { - c->set_output(0, c->Vector(3)); - c->set_output(1, c->Vector(3)); - c->set_output(2, c->MakeShape({1, 1, 4})); - return Status::OK(); - }) - .Doc(R"doc( + .Input("image_size: T") + .Input("bounding_boxes: float") + .Input("min_object_covered: float") + .Output("begin: T") + .Output("size: T") + .Output("bboxes: float") + .Attr("T: {uint8, int8, int16, int32, int64}") + .Attr("seed: int = 0") + .Attr("seed2: int = 0") + .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]") + .Attr("area_range: list(float) = [0.05, 1.0]") + .Attr("max_attempts: int = 100") + .Attr("use_image_if_no_bounding_boxes: bool = false") + .SetIsStateful() + .SetShapeFn([](InferenceContext* c) { + c->set_output(0, c->Vector(3)); + c->set_output(1, c->Vector(3)); + c->set_output(2, c->MakeShape({1, 1, 4})); + return Status::OK(); + }) + .Doc(R"doc( Generate a single randomly distorted bounding box for an image. Bounding box annotations are often supplied in addition to ground-truth labels @@ -1068,7 +1097,7 @@ REGISTER_OP("CropAndResize") .Input("box_ind: int32") .Input("crop_size: int32") .Output("crops: float") - .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}") + .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}") .Attr("method: {'bilinear'} = 'bilinear'") .Attr("extrapolation_value: float = 0") .SetShapeFn([](InferenceContext* c) { @@ -1175,7 +1204,7 @@ REGISTER_OP("CropAndResizeGradBoxes") .Input("boxes: float") .Input("box_ind: int32") .Output("output: float") - .Attr("T: {uint8, int8, int16, int32, int64, half, float, double}") + .Attr("T: {uint8, uint16, int8, int16, int32, int64, half, float, double}") .Attr("method: {'bilinear'} = 'bilinear'") .SetShapeFn([](InferenceContext* c) { c->set_output(0, c->input(2)); @@ -1207,16 +1236,16 @@ method: A string specifying the interpolation method. Only 'bilinear' is // -------------------------------------------------------------------------- REGISTER_OP("NonMaxSuppression") - .Input("boxes: float") - .Input("scores: float") - .Input("max_output_size: int32") - .Output("selected_indices: int32") - .Attr("iou_threshold: float = 0.5") - .SetShapeFn([](InferenceContext* c) { + .Input("boxes: float") + .Input("scores: float") + .Input("max_output_size: int32") + .Output("selected_indices: int32") + .Attr("iou_threshold: float = 0.5") + .SetShapeFn([](InferenceContext* c) { c->set_output(0, c->Vector(c->UnknownDim())); return Status::OK(); }) - .Doc(R"doc( + .Doc(R"doc( Greedily selects a subset of bounding boxes in descending order of score, pruning away boxes that have high intersection-over-union (IOU) overlap with previously selected boxes. Bounding boxes are supplied as diff --git a/tensorflow/core/ops/image_ops_test.cc b/tensorflow/core/ops/image_ops_test.cc index c34b11a15e0c5e041e098aceda605b42a27aa04b..5f0b391b0d14a94e687b2ebe26d4aac8d459b0df 100644 --- a/tensorflow/core/ops/image_ops_test.cc +++ b/tensorflow/core/ops/image_ops_test.cc @@ -105,7 +105,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) { .Input({"img", 0, DT_STRING}) .Input({"crop_window", 1, DT_INT32}) .Finalize(&op.node_def)); - INFER_OK(op, "[];[]", "[?,?,?]"); + INFER_OK(op, "[];[?]", "[?,?,?]"); // Set the channel, so that part of output shape is known. TF_ASSERT_OK(NodeDefBuilder("test", op_name) @@ -113,7 +113,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_ShapeFn) { .Input({"crop_window", 1, DT_INT32}) .Attr("channels", 4) .Finalize(&op.node_def)); - INFER_OK(op, "[];[]", "[?,?,4]"); + INFER_OK(op, "[];[?]", "[?,?,4]"); // Negative channel value is rejected. TF_ASSERT_OK(NodeDefBuilder("test", op_name) @@ -139,7 +139,7 @@ TEST(ImageOpsTest, DecodeAndCropJpeg_InvalidCropWindow) { .Input({"img", 0, DT_STRING}) .Input({"crop_window", 1, DT_INT32}) .Finalize(&op.node_def)); - INFER_OK(op, "[];[]", "[?,?,?]"); + INFER_OK(op, "[];[?]", "[?,?,?]"); } TEST(ImageOpsTest, EncodeImage_ShapeFn) { diff --git a/tensorflow/core/ops/linalg_ops.cc b/tensorflow/core/ops/linalg_ops.cc index 76e2149522f277aa9d6c152327c291dad81b4c07..4851619f833beff71e1976e63a60cd81fb78eff8 100644 --- a/tensorflow/core/ops/linalg_ops.cc +++ b/tensorflow/core/ops/linalg_ops.cc @@ -25,7 +25,6 @@ using shape_inference::ShapeHandle; namespace { - // Return in the result of making the end of a square matrix. Status MakeBatchSquareMatrix(InferenceContext* c, ShapeHandle input, ShapeHandle* out) { diff --git a/tensorflow/core/ops/math_grad_test.cc b/tensorflow/core/ops/math_grad_test.cc index 2b4b35547bb434bd5de119a89934bca07468f721..8dcd3e815f3c19b41b1ef02a23e1f5ce36697a23 100644 --- a/tensorflow/core/ops/math_grad_test.cc +++ b/tensorflow/core/ops/math_grad_test.cc @@ -385,7 +385,7 @@ class TestOp : public OpKernel { REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_CPU), TestOp); #ifdef TENSORFLOW_USE_SYCL REGISTER_KERNEL_BUILDER(Name("TestOpWithNoGrad").Device(DEVICE_SYCL), TestOp); -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TEST_F(MathGradTest, Error_Reporting) { auto x = test::AsTensor({-3.f}); @@ -557,11 +557,10 @@ TEST_F(MathGradTest, Acosh) { TEST_F(MathGradTest, Atanh) { auto x = test::AsTensor({-0.3f, -0.2f, -0.1f, 0.1f, 0.2f, 0.3f}, TensorShape({2, 3})); - auto g = [](float x) { - return 1.f / (1.f - x * x); - }; + auto g = [](float x) { return 1.f / (1.f - x * x); }; auto dx = test::AsTensor( - {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)}, TensorShape({2, 3})); + {g(-0.3f), g(-0.2f), g(-0.1f), g(0.1f), g(0.2f), g(0.3f)}, + TensorShape({2, 3})); auto ans = SymGrad("Atanh", x); test::ExpectClose(ans, dx); } @@ -761,7 +760,7 @@ TEST_F(MathGradTest, Pow) { } } -//TODO{lukeiwanski}: Implement Complex Pow for SYCL +// TODO{lukeiwanski}: Implement Complex Pow for SYCL #ifndef TENSORFLOW_USE_SYCL TEST_F(MathGradTest, ComplexPow) { auto x = test::AsTensor({0.f, 2.f, -2.f}, TensorShape({3})); @@ -781,7 +780,7 @@ TEST_F(MathGradTest, ComplexPow) { dy, test::AsTensor({h(0.f, 2.f), h(2.f, 2.f), h(-2.f, 2.f)}, TensorShape({3}))); } -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TEST_F(MathGradTest, Maximum) { auto x = test::AsTensor({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, @@ -943,7 +942,7 @@ TEST_F(MathGradTest, MatMul_11) { test::ExpectClose(dy, MatMul(dz, true, x, true)); } -//TODO{lukeiwanski}: Implement BatchMatMul for SYCL +// TODO{lukeiwanski}: Implement BatchMatMul for SYCL #ifndef TENSORFLOW_USE_SYCL TEST_F(MathGradTest, BatchMatMul_00) { auto x = test::AsTensor({1.f, 2.f, 3.f, 4.f, 5.f, 6.f}, @@ -992,7 +991,7 @@ TEST_F(MathGradTest, BatchMatMul_11) { test::ExpectClose(dx, BatchMatMul(y, true, dz, true)); test::ExpectClose(dy, BatchMatMul(dz, true, x, true)); } -#endif // TENSORFLOW_USE_SYCL +#endif // TENSORFLOW_USE_SYCL TEST_F(MathGradTest, Sum_dim0) { auto x = test::AsTensor({-3.f, -2.f, -1.f, 1.f, 2.f, 3.f}, diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 74af6f7f4a52ea35e801f3fa4a018de56e693151..7b10af9f44dad7c9a28a7c37d57b3b5a69cc36a1 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -235,7 +235,9 @@ value is computed as \\( \sqrt{a^2 + b^2}\\). .Attr("T: {half, float, double, complex64, complex128}") \ .SetShapeFn(shape_inference::UnchangedShape) -REGISTER_OP("Neg").UNARY().Doc(R"doc( +REGISTER_OP("Neg") + .UNARY() + .Doc(R"doc( Computes numerical negative value element-wise. I.e., \\(y = -x\\). )doc"); @@ -258,155 +260,217 @@ is the corresponding input gradient. )doc") .Deprecated(17, "Use ReciprocalGrad"); -REGISTER_OP("Reciprocal").UNARY().Doc(R"doc( +REGISTER_OP("Reciprocal") + .UNARY() + .Doc(R"doc( Computes the reciprocal of x element-wise. I.e., \\(y = 1 / x\\). )doc"); -REGISTER_OP("ReciprocalGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("ReciprocalGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the inverse of `x` wrt its input. Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Square").UNARY().Doc(R"doc( +REGISTER_OP("Square") + .UNARY() + .Doc(R"doc( Computes square of x element-wise. I.e., \\(y = x * x = x^2\\). )doc"); -REGISTER_OP("Sqrt").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sqrt") + .UNARY_COMPLEX() + .Doc(R"doc( Computes square root of x element-wise. I.e., \\(y = \sqrt{x} = x^{1/2}\\). )doc"); -REGISTER_OP("SqrtGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("SqrtGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the sqrt of `x` wrt its input. Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Rsqrt").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Rsqrt") + .UNARY_COMPLEX() + .Doc(R"doc( Computes reciprocal of square root of x element-wise. I.e., \\(y = 1 / \sqrt{x}\\). )doc"); -REGISTER_OP("Round").UNARY().Doc(R"doc( +REGISTER_OP("Round") + .UNARY() + .Doc(R"doc( Rounds the values of a tensor to the nearest integer, element-wise. Rounds half to even. Also known as bankers rounding. If you want to round according to the current system rounding mode use std::cint. )doc"); -REGISTER_OP("RsqrtGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("RsqrtGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the rsqrt of `x` wrt its input. Specifically, `grad = dy * -0.5 * y^3`, where `y = rsqrt(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Exp").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Exp") + .UNARY_COMPLEX() + .Doc(R"doc( Computes exponential of x element-wise. \\(y = e^x\\). )doc"); -REGISTER_OP("Expm1").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Expm1") + .UNARY_COMPLEX() + .Doc(R"doc( Computes exponential of x - 1 element-wise. I.e., \\(y = (\exp x) - 1\\). )doc"); -REGISTER_OP("Log").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Log") + .UNARY_COMPLEX() + .Doc(R"doc( Computes natural logarithm of x element-wise. I.e., \\(y = \log_e x\\). )doc"); -REGISTER_OP("Log1p").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Log1p") + .UNARY_COMPLEX() + .Doc(R"doc( Computes natural logarithm of (1 + x) element-wise. I.e., \\(y = \log_e (1 + x)\\). )doc"); -REGISTER_OP("Sinh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sinh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes hyperbolic sine of x element-wise. )doc"); -REGISTER_OP("Cosh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Cosh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes hyperbolic cosine of x element-wise. )doc"); -REGISTER_OP("Tanh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Tanh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes hyperbolic tangent of `x` element-wise. )doc"); -REGISTER_OP("Asinh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Asinh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes inverse hyperbolic sine of x element-wise. )doc"); -REGISTER_OP("Acosh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Acosh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes inverse hyperbolic cosine of x element-wise. )doc"); -REGISTER_OP("Atanh").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Atanh") + .UNARY_COMPLEX() + .Doc(R"doc( Computes inverse hyperbolic tangent of x element-wise. )doc"); -REGISTER_OP("TanhGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("TanhGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient for the tanh of `x` wrt its input. Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Lgamma").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Lgamma") + .UNARY_REAL() + .Doc(R"doc( Computes the log of the absolute value of `Gamma(x)` element-wise. )doc"); -REGISTER_OP("Digamma").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Digamma") + .UNARY_REAL() + .Doc(R"doc( Computes Psi, the derivative of Lgamma (the log of the absolute value of `Gamma(x)`), element-wise. )doc"); -REGISTER_OP("Erf").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Erf") + .UNARY_REAL() + .Doc(R"doc( Computes the Gauss error function of `x` element-wise. )doc"); -REGISTER_OP("Erfc").UNARY_REAL().Doc(R"doc( +REGISTER_OP("Erfc") + .UNARY_REAL() + .Doc(R"doc( Computes the complementary error function of `x` element-wise. )doc"); -REGISTER_OP("Sigmoid").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sigmoid") + .UNARY_COMPLEX() + .Doc(R"doc( Computes sigmoid of `x` element-wise. Specifically, `y = 1 / (1 + exp(-x))`. )doc"); -REGISTER_OP("SigmoidGrad").UNARY_GRADIENT_COMPLEX().Doc(R"doc( +REGISTER_OP("SigmoidGrad") + .UNARY_GRADIENT_COMPLEX() + .Doc(R"doc( Computes the gradient of the sigmoid of `x` wrt its input. Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and `dy` is the corresponding input gradient. )doc"); -REGISTER_OP("Sin").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Sin") + .UNARY_COMPLEX() + .Doc(R"doc( Computes sin of x element-wise. )doc"); -REGISTER_OP("Cos").UNARY_COMPLEX().Doc(R"doc( +REGISTER_OP("Cos") + .UNARY_COMPLEX() + .Doc(R"doc( Computes cos of x element-wise. )doc"); -REGISTER_OP("Tan").UNARY().Doc(R"doc( +REGISTER_OP("Tan") + .UNARY() + .Doc(R"doc( Computes tan of x element-wise. )doc"); -REGISTER_OP("Asin").UNARY().Doc(R"doc( +REGISTER_OP("Asin") + .UNARY() + .Doc(R"doc( Computes asin of x element-wise. )doc"); -REGISTER_OP("Acos").UNARY().Doc(R"doc( +REGISTER_OP("Acos") + .UNARY() + .Doc(R"doc( Computes acos of x element-wise. )doc"); -REGISTER_OP("Atan").UNARY().Doc(R"doc( +REGISTER_OP("Atan") + .UNARY() + .Doc(R"doc( Computes atan of x element-wise. )doc"); @@ -514,7 +578,6 @@ rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.] Input("x: T").Input("y: T").Output("z: T").Attr( \ "T: {half, float, double, int32, int64, complex64, complex128}") -// TODO(mrry): Restore `SetIsCommutative()` for non-string types. REGISTER_OP("Add") .Input("x: T") .Input("y: T") @@ -530,6 +593,25 @@ Returns x + y element-wise. [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); +// TODO(rmlarsen): Add a Python wrapper that swiches non-string instances to +// use AddV2 (b/68646025). +REGISTER_OP("AddV2") + .Input("x: T") + .Input("y: T") + .Output("z: T") + .Attr( + "T: {half, float, double, uint8, int8, int16, int32, int64, complex64, " + "complex128}") + .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) + .SetIsAggregate() + .SetIsCommutative() + .Doc(R"doc( +Returns x + y element-wise. + +*NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting +[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +)doc"); + REGISTER_OP("_MklAdd") .Input("x: T") .Input("y: T") @@ -623,7 +705,7 @@ REGISTER_OP("TruncateDiv") Returns x / y element-wise for integer types. Truncation designates that negative numbers will round fractional quantities -toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different +toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different than Python semantics. See `FloorDiv` for a division function that matches Python Semantics. @@ -942,28 +1024,36 @@ beta function. .Attr("T: realnumbertype") \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) -REGISTER_OP("Less").COMPARISON().Doc(R"doc( +REGISTER_OP("Less") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x < y) element-wise. *NOTE*: `Less` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("LessEqual").COMPARISON().Doc(R"doc( +REGISTER_OP("LessEqual") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x <= y) element-wise. *NOTE*: `LessEqual` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("Greater").COMPARISON().Doc(R"doc( +REGISTER_OP("Greater") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x > y) element-wise. *NOTE*: `Greater` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("GreaterEqual").COMPARISON().Doc(R"doc( +REGISTER_OP("GreaterEqual") + .COMPARISON() + .Doc(R"doc( Returns the truth value of (x >= y) element-wise. *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting @@ -985,14 +1075,18 @@ Returns the truth value of (x >= y) element-wise. "quint8, qint8, qint32, string, bool, complex128}") \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) -REGISTER_OP("Equal").EQUALITY_COMPARISON().Doc(R"doc( +REGISTER_OP("Equal") + .EQUALITY_COMPARISON() + .Doc(R"doc( Returns the truth value of (x == y) element-wise. *NOTE*: `Equal` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("NotEqual").EQUALITY_COMPARISON().Doc(R"doc( +REGISTER_OP("NotEqual") + .EQUALITY_COMPARISON() + .Doc(R"doc( Returns the truth value of (x != y) element-wise. *NOTE*: `NotEqual` supports broadcasting. More about broadcasting @@ -1030,14 +1124,18 @@ Returns the truth value of NOT x element-wise. .SetIsCommutative() \ .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) -REGISTER_OP("LogicalAnd").BINARY_LOGICAL().Doc(R"doc( +REGISTER_OP("LogicalAnd") + .BINARY_LOGICAL() + .Doc(R"doc( Returns the truth value of x AND y element-wise. *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) )doc"); -REGISTER_OP("LogicalOr").BINARY_LOGICAL().Doc(R"doc( +REGISTER_OP("LogicalOr") + .BINARY_LOGICAL() + .Doc(R"doc( Returns the truth value of x OR y element-wise. *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting @@ -1977,12 +2075,12 @@ Status RangeSize(const Tensor* start_t, const Tensor* limit_t, T limit = limit_t->scalar()(); T delta = delta_t->scalar()(); if (start > limit && delta > 0) { - return errors::InvalidArgument( - "Requires start <= limit when delta > 0: ", start, "/", limit); + return errors::InvalidArgument("Requires start <= limit when delta > 0: ", + start, "/", limit); } if (start < limit && delta < 0) { - return errors::InvalidArgument( - "Requires start >= limit when delta < 0: ", start, "/", limit); + return errors::InvalidArgument("Requires start >= limit when delta < 0: ", + start, "/", limit); } if (delta == 0) { return errors::InvalidArgument("Requires delta != 0"); @@ -2250,6 +2348,51 @@ product: Pairwise cross product of the vectors in `a` and `b`. // -------------------------------------------------------------------------- +REGISTER_OP("HistogramFixedWidth") + .Input("values: T") + .Input("value_range: T") + .Input("nbins: int32") + .Output("out: dtype") + .Attr("T: {int32, int64, float32, float64}") + .Attr("dtype: {int32, int64} = DT_INT32") + .SetShapeFn([](InferenceContext* c) { + const Tensor* nbins_input = c->input_tensor(2); + if (nbins_input != nullptr) { + int64 nbins; + TF_RETURN_IF_ERROR(c->GetScalarFromTensor(nbins_input, &nbins)); + c->set_output(0, c->Vector(nbins)); + } else { + c->set_output(0, c->UnknownShapeOfRank(1)); + } + return Status::OK(); + }) + .Doc(R"doc( +Return histogram of values. + +Given the tensor `values`, this operation returns a rank 1 histogram counting +the number of entries in `values` that fall into every bin. The bins are +equal width and determined by the arguments `value_range` and `nbins`. + +```python +# Bins will be: (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf) +nbins = 5 +value_range = [0.0, 5.0] +new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] + +with tf.get_default_session() as sess: + hist = tf.histogram_fixed_width(new_values, value_range, nbins=5) + variables.global_variables_initializer().run() + sess.run(hist) => [2, 1, 1, 0, 2] +``` + +values: Numeric `Tensor`. +value_range: Shape [2] `Tensor` of same `dtype` as `values`. + values <= value_range[0] will be mapped to hist[0], + values >= value_range[1] will be mapped to hist[-1]. +nbins: Scalar `int32 Tensor`. Number of histogram bins. +out: A 1-D `Tensor` holding histogram of values. +)doc"); + REGISTER_OP("Bincount") .Input("arr: int32") .Input("size: int32") diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc index 5efa55b4960fc985df381756a60279b031429e2c..de059a3e7ef2f4a732df27bff86cad79edd53541 100644 --- a/tensorflow/core/ops/nn_ops.cc +++ b/tensorflow/core/ops/nn_ops.cc @@ -2176,9 +2176,9 @@ Status TopKShapeFn(InferenceContext* c) { DimensionHandle last_dim = c->Dim(input, -1); if (c->ValueKnown(last_dim) && c->ValueKnown(k_dim) && c->Value(last_dim) < c->Value(k_dim)) { - return errors::InvalidArgument( - "input must have last dimension >= k = ", c->Value(k_dim), " but is ", - c->Value(last_dim)); + return errors::InvalidArgument("input must have last dimension >= k = ", + c->Value(k_dim), " but is ", + c->Value(last_dim)); } // Replace last_dim with k_dim. @@ -2260,6 +2260,56 @@ indices: The indices of `values` within the last dimension of `input`. // -------------------------------------------------------------------------- +REGISTER_OP("NthElement") + .Input("input: T") + .Input("n: int32") + .Output("values: T") + .Attr("reverse: bool = false") + .Attr("T: realnumbertype") + .SetShapeFn([](InferenceContext* c) { + ShapeHandle input; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &input)); + + // Get the n value from input tensor, and make sure which is a scalar. + DimensionHandle n_dim; + TF_RETURN_IF_ERROR(c->MakeDimForScalarInput(1, &n_dim)); + + // The last dimension of input tensor must be greater than N. + DimensionHandle last_dim = c->Dim(input, -1); + if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) && + c->Value(last_dim) <= c->Value(n_dim)) { + return errors::InvalidArgument("Input must have last dimension > n = ", + c->Value(n_dim), " but is ", + c->Value(last_dim)); + } + + // Reduce last_dim for output tensor + ShapeHandle s; + TF_RETURN_IF_ERROR(c->Subshape(input, 0, -1, &s)); + c->set_output(0, s); + return Status::OK(); + }) + .Doc(R"doc( +Finds values of the `n`-th order statistic for the last dmension. + +If the input is a vector (rank-1), finds the entries which is the nth-smallest +value in the vector and outputs their values as scalar tensor. + +For matrices (resp. higher rank input), computes the entries which is the +nth-smallest value in each row (resp. vector along the last dimension). Thus, + + values.shape = input.shape[:-1] + +input: 1-D or higher with last dimension at least `n+1`. +n: 0-D. Position of sorted vector to select along the last dimension (along + each row for matrices). Valid range of n is `[0, input.shape[:-1])` +reverse: When set to True, find the nth-largest value in the vector and vice + versa. +values: The `n`-th order statistic along each last dimensional slice. +)doc"); + +// -------------------------------------------------------------------------- + REGISTER_OP("FractionalMaxPool") .Input("value: T") .Output("output: T") diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc index 4628b725f8c74d0e422f232274bf5f7d283860a2..1b17a7cda65f210e1981e0f46f47691f0faba465 100644 --- a/tensorflow/core/ops/nn_ops_test.cc +++ b/tensorflow/core/ops/nn_ops_test.cc @@ -81,6 +81,29 @@ TEST(NNOpsTest, TopKV2_ShapeFn) { op, "[1,2,3,4];[]"); } +TEST(NNOpsTest, NthElement_ShapeFn) { + ShapeInferenceTestOp op("NthElement"); + op.input_tensors.resize(2); + + Tensor n_t; + op.input_tensors[1] = &n_t; + n_t = test::AsScalar(20); + + INFER_OK(op, "?;[]", "?"); + INFER_OK(op, "[21];[]", "[]"); + INFER_OK(op, "[2,?,?];[]", "[d0_0,d0_1]"); + INFER_OK(op, "[?,3,?,21];[]", "[d0_0,d0_1,d0_2]"); + + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, "[];[]"); + INFER_ERROR("Input must have last dimension > n = 20 but is 1", op, "[1];[]"); + INFER_ERROR("Input must have last dimension > n = 20 but is 20", op, + "[1,2,3,20];[]"); + n_t = test::AsScalar(-1); + INFER_ERROR( + "Dimension size, given by scalar input 1, must be non-negative but is -1", + op, "[1,2,3,4];[]"); +} + TEST(NNOpsTest, BatchNormWithGlobalNormalization_ShapeFn) { ShapeInferenceTestOp op("BatchNormWithGlobalNormalization"); @@ -362,9 +385,8 @@ TEST(NNOpsTest, Dilation2DBackpropFilter_ShapeFn) { } TEST(NNOpsTest, MergeBothInputs_ShapeFn) { - for (const char* op_name : - {"ReluGrad", "Relu6Grad", "EluGrad", "SeluGrad", "SoftplusGrad", - "SoftsignGrad"}) { + for (const char* op_name : {"ReluGrad", "Relu6Grad", "EluGrad", "SeluGrad", + "SoftplusGrad", "SoftsignGrad"}) { ShapeInferenceTestOp op(op_name); INFER_OK(op, "?;?", "in0|in1"); diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 6403dcf78c5c3ea0da4469f7ac92cb296cc7506d..4017a46521335c5f645ac9a95f4fde4d86cb642c 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -44,6 +44,58 @@ op { summary: "Computes the absolute value of a tensor." description: "Given a tensor `x`, this operation returns a tensor containing the absolute\nvalue of each element in `x`. For example, if x is an input element and y is\nan output element, this operation computes \\\\(y = |x|\\\\)." } +op { + name: "AccumulateNV2" + input_arg { + name: "inputs" + description: "A list of `Tensor` objects, each with same shape and type." + type_attr: "T" + number_attr: "N" + } + output_arg { + name: "sum" + type_attr: "T" + } + attr { + name: "N" + type: "int" + has_minimum: true + minimum: 1 + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "shape" + type: "shape" + description: "Shape of elements of `inputs`." + } + summary: "Returns the element-wise sum of a list of tensors." + description: "`tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not\nwait for all of its inputs to be ready before beginning to sum. This can\nsave memory if inputs are ready at different times, since minimum temporary\nstorage is proportional to the output size rather than the inputs size.\n\nUnlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.\n\nReturns a `Tensor` of same shape and type as the elements of `inputs`." + is_aggregate: true + is_commutative: true +} op { name: "AccumulatorApplyGradient" input_arg { @@ -394,6 +446,43 @@ op { description: "A `SparseTensor` is represented by three tensors: `sparse_indices`,\n`sparse_values`, and `sparse_shape`.\n\nThis operator takes the given `SparseTensor` and adds it to a container\nobject (a `SparseTensorsMap`). A unique key within this container is generated\nin the form of an `int64`, and this is the value that is returned.\n\nThe `SparseTensor` can then be read out as part of a minibatch by passing\nthe key as a vector element to `TakeManySparseFromTensorsMap`. To ensure\nthe correct `SparseTensorsMap` is accessed, ensure that the same\n`container` and `shared_name` are passed to that Op. If no `shared_name`\nis provided here, instead use the *name* of the Operation created by calling\n`AddSparseToTensorsMap` as the `shared_name` passed to\n`TakeManySparseFromTensorsMap`. Ensure the Operations are colocated." is_stateful: true } +op { + name: "AddV2" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_HALF + type: DT_FLOAT + type: DT_DOUBLE + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + } + } + } + summary: "Returns x + y element-wise." + description: "*NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" + is_aggregate: true + is_commutative: true +} op { name: "AdjustContrast" input_arg { @@ -4008,6 +4097,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -4040,6 +4131,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -4072,6 +4165,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -5633,6 +5728,7 @@ op { allowed_values { list { type: DT_UINT8 + type: DT_UINT16 type: DT_INT8 type: DT_INT16 type: DT_INT32 @@ -5700,6 +5796,7 @@ op { allowed_values { list { type: DT_UINT8 + type: DT_UINT16 type: DT_INT8 type: DT_INT16 type: DT_INT32 @@ -6989,7 +7086,22 @@ op { } } summary: "Dequantize the \'input\' tensor into a float Tensor." - description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6. The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / number_of_steps\nconst double offset_input = static_cast(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```" + description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nif T == qint8, in[i] += (range(T) + 1)/ 2.0\nout[i] = min_range + (in[i]* (max_range - min_range) / range(T))\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nIf the input comes from a QuantizedRelu6, the output type is\nquint8 (range of 0-255) but the possible range of QuantizedRelu6 is\n0-6. The min_range and max_range values are therefore 0.0 and 6.0.\nDequantize on quint8 will take each value, cast to float, and multiply\nby 6 / 255.\nNote that if quantizedtype is qint8, the operation will additionally add\neach value by 128 prior to casting.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```c++\nnum_discrete_values = 1 << (# of bits in T)\nrange_adjust = num_discrete_values / (num_discrete_values - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = range / num_discrete_values\nconst double offset_input = static_cast(input) - lowest_quantized;\nresult = range_min + ((input - numeric_limits::min()) * range_scale)\n```\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (2 * m) / (max_fixed - min_fixed)\n```\n\nNow we can dequantize the elements of our tensor:\n```c++\nresult = input * s\n```" +} +op { + name: "DeserializeIterator" + input_arg { + name: "resource_handle" + description: "A handle to an iterator resource." + type: DT_RESOURCE + } + input_arg { + name: "serialized" + description: "A variant tensor storing the state of the iterator contained in the\nresource." + type: DT_VARIANT + } + summary: "Converts the given variant tensor to an iterator and stores it in the given resource." + is_stateful: true } op { name: "DeserializeManySparse" @@ -7065,7 +7177,7 @@ op { name: "Diag" input_arg { name: "diagonal" - description: "Rank k tensor where k is at most 3." + description: "Rank k tensor where k is at most 1." type_attr: "T" } output_arg { @@ -7093,7 +7205,7 @@ op { name: "DiagPart" input_arg { name: "input" - description: "Rank k tensor where k is 2, 4, or 6." + description: "Rank k tensor where k is even and not zero." type_attr: "T" } output_arg { @@ -10279,6 +10391,56 @@ op { description: "This op creates a hash table, specifying the type of its keys and values.\nBefore using the table you will have to initialize it. After initialization the\ntable will be immutable." is_stateful: true } +op { + name: "HistogramFixedWidth" + input_arg { + name: "values" + description: "Numeric `Tensor`." + type_attr: "T" + } + input_arg { + name: "value_range" + description: "Shape [2] `Tensor` of same `dtype` as `values`.\nvalues <= value_range[0] will be mapped to hist[0],\nvalues >= value_range[1] will be mapped to hist[-1]." + type_attr: "T" + } + input_arg { + name: "nbins" + description: "Scalar `int32 Tensor`. Number of histogram bins." + type: DT_INT32 + } + output_arg { + name: "out" + description: "A 1-D `Tensor` holding histogram of values." + type_attr: "dtype" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + type: DT_FLOAT + type: DT_DOUBLE + } + } + } + attr { + name: "dtype" + type: "type" + default_value { + type: DT_INT32 + } + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + summary: "Return histogram of values." + description: "Given the tensor `values`, this operation returns a rank 1 histogram counting\nthe number of entries in `values` that fall into every bin. The bins are\nequal width and determined by the arguments `value_range` and `nbins`.\n\n```python\n# Bins will be: (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)\nnbins = 5\nvalue_range = [0.0, 5.0]\nnew_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]\n\nwith tf.get_default_session() as sess:\n hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)\n variables.global_variables_initializer().run()\n sess.run(hist) => [2, 1, 1, 0, 2]\n```" +} op { name: "HistogramSummary" input_arg { @@ -11083,6 +11245,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -11564,6 +11728,40 @@ op { description: "See explanations of candidate sampling and the data formats at\ngo/candidate-sampling.\n\nFor each batch, this op picks a single set of sampled candidate labels.\n\nThe advantages of sampling candidates per-batch are simplicity and the\npossibility of efficient dense matrix multiplication. The disadvantage is that\nthe sampled candidates must be chosen independently of the context and of the\ntrue labels." is_stateful: true } +op { + name: "LeftShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + summary: "Elementwise computes the bitwise left-shift of `x` and `y`." + description: "If `y` is negative, or greater than or equal to the width of `x` in bits the\nresult is implementation defined." + is_commutative: true +} op { name: "Less" input_arg { @@ -15180,6 +15378,53 @@ op { description: "*NOTE*: `NotEqual` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" is_commutative: true } +op { + name: "NthElement" + input_arg { + name: "input" + description: "1-D or higher with last dimension at least `n+1`." + type_attr: "T" + } + input_arg { + name: "n" + description: "0-D. Position of sorted vector to select along the last dimension (along\neach row for matrices). Valid range of n is `[0, input.shape[:-1])`" + type: DT_INT32 + } + output_arg { + name: "values" + description: "The `n`-th order statistic along each last dimensional slice." + type_attr: "T" + } + attr { + name: "reverse" + type: "bool" + default_value { + b: false + } + description: "When set to True, find the nth-largest value in the vector and vice\nversa." + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_INT16 + type: DT_INT8 + type: DT_UINT16 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + summary: "Finds values of the `n`-th order statistic for the last dmension." + description: "If the input is a vector (rank-1), finds the entries which is the nth-smallest\nvalue in the vector and outputs their values as scalar tensor.\n\nFor matrices (resp. higher rank input), computes the entries which is the\nnth-smallest value in each row (resp. vector along the last dimension). Thus,\n\n values.shape = input.shape[:-1]" +} op { name: "OneHot" input_arg { @@ -15973,6 +16218,57 @@ op { summary: "Interleave the values from the `data` tensors into a single tensor." description: "Builds a merged tensor such that\n\n```python\n merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]\n```\n\nFor example, if each `indices[m]` is scalar or vector, we have\n\n```python\n # Scalar indices:\n merged[indices[m], ...] = data[m][...]\n\n # Vector indices:\n merged[indices[m][i], ...] = data[m][i, ...]\n```\n\nEach `data[i].shape` must start with the corresponding `indices[i].shape`,\nand the rest of `data[i].shape` must be constant w.r.t. `i`. That is, we\nmust have `data[i].shape = indices[i].shape + constant`. In terms of this\n`constant`, the output shape is\n\n merged.shape = [max(indices)] + constant\n\nValues may be merged in parallel, so if an index appears in both `indices[m][i]`\nand `indices[n][j]`, the result may be invalid. This differs from the normal\nDynamicStitch operator that defines the behavior in that case.\n\nFor example:\n\n```python\n indices[0] = 6\n indices[1] = [4, 1]\n indices[2] = [[5, 2], [0, 3]]\n data[0] = [61, 62]\n data[1] = [[41, 42], [11, 12]]\n data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]\n merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],\n [51, 52], [61, 62]]\n```\n\nThis method can be used to merge partitions created by `dynamic_partition`\nas illustrated on the following example:\n\n```python\n # Apply function (increments x_i) on elements for which a certain condition\n # apply (x_i != -1 in this example).\n x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])\n condition_mask=tf.not_equal(x,tf.constant(-1.))\n partitioned_data = tf.dynamic_partition(\n x, tf.cast(condition_mask, tf.int32) , 2)\n partitioned_data[1] = partitioned_data[1] + 1.0\n condition_indices = tf.dynamic_partition(\n tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)\n x = tf.dynamic_stitch(condition_indices, partitioned_data)\n # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain\n # unchanged.\n```\n\n
\n\n
" } +op { + name: "ParallelInterleaveDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "other_arguments" + type_list_attr: "Targuments" + } + input_arg { + name: "cycle_length" + type: DT_INT64 + } + input_arg { + name: "block_length" + type: DT_INT64 + } + input_arg { + name: "sloppy" + type: DT_BOOL + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "f" + type: "func" + description: "A function mapping elements of `input_dataset`, concatenated with\n`other_arguments`, to a Dataset variant that contains elements matching\n`output_types` and `output_shapes`." + } + attr { + name: "Targuments" + type: "list(type)" + has_minimum: true + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`." + description: "The resulting dataset is similar to the `InterleaveDataset`, with the exception\nthat if retrieving the next value from a dataset would cause the requester to\nblock, it will skip that input dataset. This dataset is especially useful\nwhen loading data from a variable-latency datastores (e.g. HDFS, GCS), as it\nallows the training step to proceed so long as some data is available.\n\n!! WARNING !! This dataset is not deterministic!" +} op { name: "ParallelMapDataset" input_arg { @@ -16522,6 +16818,8 @@ op { type: DT_INT64 type: DT_UINT8 type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 } } } @@ -17225,8 +17523,21 @@ op { } } } + attr { + name: "round_mode" + type: "string" + default_value { + s: "HALF_AWAY_FROM_ZERO" + } + allowed_values { + list { + s: "HALF_AWAY_FROM_ZERO" + s: "HALF_TO_EVEN" + } + } + } summary: "Quantize the \'input\' tensor of type float to \'output\' tensor of type \'T\'." - description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnumber_of_steps = 1 << (# of bits in T)\nrange_adjust = number_of_steps / (number_of_steps - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = number_of_steps / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n numeric_limits::min()\nquantized = max(quantized, numeric_limits::min())\nquantized = min(quantized, numeric_limits::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = (input * s).round_to_nearest()\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them." + description: "[min_range, max_range] are scalar floats that specify the range for\nthe \'input\' data. The \'mode\' attribute controls exactly which calculations are\nused to convert the float values to their quantized equivalents. The\n\'round_mode\' attribute controls which rounding tie-breaking algorithm is used\nwhen rounding float values to their quantized equivalents.\n\nIn \'MIN_COMBINED\' mode, each value of the tensor will undergo the following:\n\n```\nout[i] = (in[i] - min_range) * range(T) / (max_range - min_range)\nif T == qint8, out[i] -= (range(T) + 1) / 2.0\n```\nhere `range(T) = numeric_limits::max() - numeric_limits::min()`\n\n*MIN_COMBINED Mode Example*\n\nAssume the input is type float and has a possible range of [0.0, 6.0] and the\noutput type is quint8 ([0, 255]). The min_range and max_range values should be\nspecified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each\nvalue of the input by 255/6 and cast to quint8.\n\nIf the output type was qint8 ([-128, 127]), the operation will additionally\nsubtract each value by 128 prior to casting, so that the range of values aligns\nwith the range of qint8.\n\nIf the mode is \'MIN_FIRST\', then this approach is used:\n\n```\nnum_discrete_values = 1 << (# of bits in T)\nrange_adjust = num_discrete_values / (num_discrete_values - 1)\nrange = (range_max - range_min) * range_adjust\nrange_scale = num_discrete_values / range\nquantized = round(input * range_scale) - round(range_min * range_scale) +\n numeric_limits::min()\nquantized = max(quantized, numeric_limits::min())\nquantized = min(quantized, numeric_limits::max())\n```\n\nThe biggest difference between this and MIN_COMBINED is that the minimum range\nis rounded first, before it\'s subtracted from the rounded value. With\nMIN_COMBINED, a small bias is introduced where repeated iterations of quantizing\nand dequantizing will introduce a larger and larger error.\n\n*SCALED mode Example*\n\n`SCALED` mode matches the quantization approach used in\n`QuantizeAndDequantize{V2|V3}`.\n\nIf the mode is `SCALED`, we do not use the full range of the output type,\nchoosing to elide the lowest possible value for symmetry (e.g., output range is\n-127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to\n0.\n\nWe first find the range of values in our tensor. The\nrange we use is always centered on 0, so we find m such that\n```c++\n m = max(abs(input_min), abs(input_max))\n```\n\nOur input tensor range is then `[-m, m]`.\n\nNext, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.\nIf T is signed, this is\n```\n num_bits = sizeof(T) * 8\n [min_fixed, max_fixed] =\n [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]\n```\n\nOtherwise, if T is unsigned, the fixed-point range is\n```\n [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]\n```\n\nFrom this we compute our scaling factor, s:\n```c++\n s = (max_fixed - min_fixed) / (2 * m)\n```\n\nNow we can quantize the elements of our tensor:\n```c++\nresult = round(input * s)\n```\n\nOne thing to watch out for is that the operator may choose to adjust the\nrequested minimum and maximum values slightly during the quantization process,\nso you should always use the output ports as the range for further calculations.\nFor example, if the requested minimum and maximum values are close to equal,\nthey will be separated by a small epsilon value to prevent ill-formed quantized\nbuffers from being created. Otherwise, you can end up with buffers where all the\nquantized values map to the same float value, which causes problems for\noperations that have to perform further calculations on them." } op { name: "QuantizedAdd" @@ -20755,9 +21066,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -20799,9 +21111,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -20880,9 +21193,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -20962,9 +21276,10 @@ op { type: "type" allowed_values { list { - type: DT_UINT8 type: DT_INT8 + type: DT_UINT8 type: DT_INT16 + type: DT_UINT16 type: DT_INT32 type: DT_INT64 type: DT_HALF @@ -21901,6 +22216,36 @@ op { description: "Note that in dense implementation of this algorithm, ms and mom will\nupdate even if the grad is zero, but in this sparse implementation, ms\nand mom will not update in iterations during which the grad is zero.\n\nmean_square = decay * mean_square + (1-decay) * gradient ** 2\nDelta = learning_rate * gradient / sqrt(mean_square + epsilon)\n\nms <- rho * ms_{t-1} + (1-rho) * grad * grad\nmom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)\nvar <- var - mom" is_stateful: true } +op { + name: "ResourceCountUpTo" + input_arg { + name: "resource" + description: "Should be from a scalar `Variable` node." + type: DT_RESOURCE + } + output_arg { + name: "output" + description: "A copy of the input before increment. If nothing else modifies the\ninput, the values produced will all be distinct." + type_attr: "T" + } + attr { + name: "limit" + type: "int" + description: "If incrementing ref would bring it above limit, instead generates an\n\'OutOfRange\' error." + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + summary: "Increments variable pointed to by \'resource\' until it reaches \'limit\'." + is_stateful: true +} op { name: "ResourceGather" input_arg { @@ -21992,7 +22337,62 @@ op { } } summary: "Adds sparse updates to the variable referenced by `resource`." - description: "This operation computes\n\n # Scalar indices\n ref[indices, ...] += updates[...]\n\n # Vector indices (for each i)\n ref[indices[i], ...] += updates[i, ...]\n\n # High rank indices (for each i, ..., j)\n ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]\n\nDuplicate entries are handled correctly: if multiple `indices` reference\nthe same location, their contributions add.\n\nRequires `updates.shape = indices.shape + ref.shape[1:]`.\n\n
\n\n
" + description: "This operation computes\n\n # Scalar indices\n ref[indices, ...] += updates[...]\n\n # Vector indices (for each i)\n ref[indices[i], ...] += updates[i, ...]\n\n # High rank indices (for each i, ..., j)\n ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]\n\nDuplicate entries are handled correctly: if multiple `indices` reference\nthe same location, their contributions add.\n\nRequires `updates.shape = indices.shape + ref.shape[1:]`.\n\n
\n\n
" + is_stateful: true +} +op { + name: "ResourceScatterUpdate" + input_arg { + name: "resource" + description: "Should be from a `Variable` node." + type: DT_RESOURCE + } + input_arg { + name: "indices" + description: "A tensor of indices into the first dimension of `ref`." + type_attr: "Tindices" + } + input_arg { + name: "updates" + description: "A tensor of updated values to add to `ref`." + type_attr: "dtype" + } + attr { + name: "dtype" + type: "type" + allowed_values { + list { + type: DT_FLOAT + type: DT_DOUBLE + type: DT_INT64 + type: DT_INT32 + type: DT_UINT8 + type: DT_UINT16 + type: DT_INT16 + type: DT_INT8 + type: DT_COMPLEX64 + type: DT_COMPLEX128 + type: DT_QINT8 + type: DT_QUINT8 + type: DT_QINT32 + type: DT_HALF + type: DT_UINT32 + type: DT_UINT64 + } + } + } + attr { + name: "Tindices" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + summary: "Assigns sparse updates to the variable referenced by `resource`." + description: "This operation computes\n\n # Scalar indices\n ref[indices, ...] = updates[...]\n\n # Vector indices (for each i)\n ref[indices[i], ...] = updates[i, ...]\n\n # High rank indices (for each i, ..., j)\n ref[indices[i, ..., j], ...] = updates[i, ..., j, ...]" is_stateful: true } op { @@ -22981,19 +23381,6 @@ op { description: "Reads a tensor stored in one or several files. If there are several files (for\ninstance because a tensor was saved as slices), `file_pattern` may contain\nwildcard symbols (`*` and `?`) in the filename portion only, not in the\ndirectory portion.\n\nIf a `file_pattern` matches several files, `preferred_shard` can be used to hint\nin which file the requested tensor is likely to be found. This op will first\nopen the file at index `preferred_shard` in the list of matching files and try\nto restore tensors from that file. Only if some tensors or tensor slices are\nnot found in that first file, then the Op opens all the files. Setting\n`preferred_shard` to match the value passed as the `shard` input\nof a matching `Save` Op may speed up Restore. This attribute only affects\nperformance, not correctness. The default value -1 means files are processed in\norder.\n\nSee also `RestoreSlice`." is_stateful: true } -op { - name: "RestoreIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - summary: "Restores the state of the `iterator` from the checkpoint saved at `path` using \"SaveIterator\"." - is_stateful: true -} op { name: "RestoreSlice" input_arg { @@ -23209,7 +23596,41 @@ op { } } summary: "Reverses specific dimensions of a tensor." - description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0, 1, 2, 3],\n# [ 4, 5, 6, 7],\n# [ 8, 9, 10, 11]],\n# [[12, 13, 14, 15],\n# [16, 17, 18, 19],\n# [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is -1\nreverse(t, dims) ==> [[[[ 3, 2, 1, 0],\n [ 7, 6, 5, 4],\n [ 11, 10, 9, 8]],\n [[15, 14, 13, 12],\n [19, 18, 17, 16],\n [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n [16, 17, 18, 19],\n [20, 21, 22, 23]\n [[ 0, 1, 2, 3],\n [ 4, 5, 6, 7],\n [ 8, 9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n [4, 5, 6, 7],\n [0, 1, 2, 3]]\n [[20, 21, 22, 23],\n [16, 17, 18, 19],\n [12, 13, 14, 15]]]]\n```" + description: "NOTE `tf.reverse` has now changed behavior in preparation for 1.0.\n`tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.\n\nGiven a `tensor`, and a `int32` tensor `axis` representing the set of\ndimensions of `tensor` to reverse. This operation reverses each dimension\n`i` for which there exists `j` s.t. `axis[j] == i`.\n\n`tensor` can have up to 8 dimensions. The number of dimensions specified\nin `axis` may be 0 or more entries. If an index is specified more than\nonce, a InvalidArgument error is raised.\n\nFor example:\n\n```\n# tensor \'t\' is [[[[ 0, 1, 2, 3],\n# [ 4, 5, 6, 7],\n# [ 8, 9, 10, 11]],\n# [[12, 13, 14, 15],\n# [16, 17, 18, 19],\n# [20, 21, 22, 23]]]]\n# tensor \'t\' shape is [1, 2, 3, 4]\n\n# \'dims\' is [3] or \'dims\' is [-1]\nreverse(t, dims) ==> [[[[ 3, 2, 1, 0],\n [ 7, 6, 5, 4],\n [ 11, 10, 9, 8]],\n [[15, 14, 13, 12],\n [19, 18, 17, 16],\n [23, 22, 21, 20]]]]\n\n# \'dims\' is \'[1]\' (or \'dims\' is \'[-3]\')\nreverse(t, dims) ==> [[[[12, 13, 14, 15],\n [16, 17, 18, 19],\n [20, 21, 22, 23]\n [[ 0, 1, 2, 3],\n [ 4, 5, 6, 7],\n [ 8, 9, 10, 11]]]]\n\n# \'dims\' is \'[2]\' (or \'dims\' is \'[-2]\')\nreverse(t, dims) ==> [[[[8, 9, 10, 11],\n [4, 5, 6, 7],\n [0, 1, 2, 3]]\n [[20, 21, 22, 23],\n [16, 17, 18, 19],\n [12, 13, 14, 15]]]]\n```" +} +op { + name: "RightShift" + input_arg { + name: "x" + type_attr: "T" + } + input_arg { + name: "y" + type_attr: "T" + } + output_arg { + name: "z" + type_attr: "T" + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + type: DT_UINT8 + type: DT_UINT16 + type: DT_UINT32 + type: DT_UINT64 + } + } + } + summary: "Elementwise computes the bitwise right-shift of `x` and `y`." + description: "Performs a logical shift for unsigned integer types, and an arithmetic shift\nfor signed integer types.\n\nIf `y` is negative, or greater than or equal to than the width of `x` in bits\nthe result is implementation defined." + is_commutative: true } op { name: "Rint" @@ -23554,20 +23975,6 @@ op { description: "The size of `tensor_names` must match the number of tensors in `data`. `data[i]`\nis written to `filename` with name `tensor_names[i]`.\n\nSee also `SaveSlices`." is_stateful: true } -op { - name: "SaveIterator" - input_arg { - name: "iterator" - type: DT_RESOURCE - } - input_arg { - name: "path" - type: DT_STRING - } - summary: "Saves the state of the `iterator` at `path`." - description: "This state can be restored using \"RestoreIterator\"." - is_stateful: true -} op { name: "SaveSlices" input_arg { @@ -24912,6 +25319,21 @@ op { } summary: "Computes gradients for the scaled exponential linear (Selu) operation." } +op { + name: "SerializeIterator" + input_arg { + name: "resource_handle" + description: "A handle to an iterator resource." + type: DT_RESOURCE + } + output_arg { + name: "serialized" + description: "A variant tensor storing the state of the iterator contained in the\nresource." + type: DT_VARIANT + } + summary: "Converts the given `resource_handle` representing an iterator to a variant tensor." + is_stateful: true +} op { name: "SerializeManySparse" input_arg { @@ -25493,53 +25915,6 @@ op { summary: "Return a slice from \'input\'." description: "The output tensor is a tensor with dimensions described by \'size\'\nwhose values are extracted from \'input\' starting at the offsets in\n\'begin\'.\n\n*Requirements*:\n 0 <= begin[i] <= begin[i] + size[i] <= Di for i in [0, n)" } -op { - name: "SloppyInterleaveDataset" - input_arg { - name: "input_dataset" - type: DT_VARIANT - } - input_arg { - name: "other_arguments" - type_list_attr: "Targuments" - } - input_arg { - name: "cycle_length" - type: DT_INT64 - } - input_arg { - name: "block_length" - type: DT_INT64 - } - output_arg { - name: "handle" - type: DT_VARIANT - } - attr { - name: "f" - type: "func" - description: "A function mapping elements of `input_dataset`, concatenated with\n`other_arguments`, to a Dataset variant that contains elements matching\n`output_types` and `output_shapes`." - } - attr { - name: "Targuments" - type: "list(type)" - has_minimum: true - } - attr { - name: "output_types" - type: "list(type)" - has_minimum: true - minimum: 1 - } - attr { - name: "output_shapes" - type: "list(shape)" - has_minimum: true - minimum: 1 - } - summary: "Creates a dataset that applies `f` to the outputs of `input_dataset`." - description: "The resulting dataset is similar to the `InterleaveDataset`, with the exception\nthat if retrieving the next value from a dataset would cause the requester to\nblock, it will skip that input dataset. This dataset is especially useful\nwhen loading data from a variable-latency datastores (e.g. HDFS, GCS), as it\nallows the training step to proceed so long as some data is available.\n\n!! WARNING !! This dataset is not deterministic!" -} op { name: "Softmax" input_arg { @@ -31984,7 +32359,7 @@ op { } } summary: "Returns x / y element-wise for integer types." - description: "Truncation designates that negative numbers will round fractional quantities\ntoward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different\nthan Python semantics. See `FloorDiv` for a division function that matches\nPython Semantics.\n\n*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" + description: "Truncation designates that negative numbers will round fractional quantities\ntoward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different\nthan Python semantics. See `FloorDiv` for a division function that matches\nPython Semantics.\n\n*NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting\n[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)" } op { name: "TruncateMod" diff --git a/tensorflow/core/ops/parsing_ops.cc b/tensorflow/core/ops/parsing_ops.cc index b44ea2e080eac3ea9ba18b6c7d640fa98931a17e..40ec792ef82ff5e0bdf6d0c4e35bf18f5560c5a7 100644 --- a/tensorflow/core/ops/parsing_ops.cc +++ b/tensorflow/core/ops/parsing_ops.cc @@ -329,7 +329,7 @@ REGISTER_OP("DecodeCSV") .Input("records: string") .Input("record_defaults: OUT_TYPE") .Output("output: OUT_TYPE") - .Attr("OUT_TYPE: list({float,int32,int64,string})") + .Attr("OUT_TYPE: list({float,double,int32,int64,string})") .Attr("field_delim: string = ','") .Attr("use_quote_delim: bool = true") .Attr("na_value: string = ''") diff --git a/tensorflow/core/ops/resource_variable_ops.cc b/tensorflow/core/ops/resource_variable_ops.cc index c4802a1cc1e980600444204742909aa2270b3e0c..cdfbec85cf1194d02c81cb4a3d66563dc85dfa57 100644 --- a/tensorflow/core/ops/resource_variable_ops.cc +++ b/tensorflow/core/ops/resource_variable_ops.cc @@ -311,7 +311,7 @@ the same location, their contributions add. Requires `updates.shape = indices.shape + ref.shape[1:]`.
- +
resource: Should be from a `Variable` node. @@ -319,4 +319,44 @@ indices: A tensor of indices into the first dimension of `ref`. updates: A tensor of updated values to add to `ref`. )doc"); +REGISTER_OP("ResourceScatterUpdate") + .Input("resource: resource") + .Input("indices: Tindices") + .Input("updates: dtype") + .Attr("dtype: numbertype") + .Attr("Tindices: {int32, int64}") + .SetShapeFn([](InferenceContext* c) { + ShapeAndType handle_shape_and_type; + TF_RETURN_IF_ERROR( + ValidateVariableResourceHandle(c, &handle_shape_and_type)); + ShapeHandle var_shape = handle_shape_and_type.shape; + ShapeHandle indices_shape = c->input(1); + + ShapeHandle unused_updates_shape; + ShapeHandle concat; + ShapeHandle var_subshape; + TF_RETURN_IF_ERROR(c->Subshape(var_shape, 1, &var_subshape)); + TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, var_subshape, &concat)); + TF_RETURN_IF_ERROR(c->Merge(c->input(2), concat, &unused_updates_shape)); + return Status::OK(); + }) + .Doc(R"doc( +Assigns sparse updates to the variable referenced by `resource`. + +This operation computes + + # Scalar indices + ref[indices, ...] = updates[...] + + # Vector indices (for each i) + ref[indices[i], ...] = updates[i, ...] + + # High rank indices (for each i, ..., j) + ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] + +resource: Should be from a `Variable` node. +indices: A tensor of indices into the first dimension of `ref`. +updates: A tensor of updated values to add to `ref`. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/core/ops/sparse_ops_test.cc b/tensorflow/core/ops/sparse_ops_test.cc index ea49f1a19919ad442d89fef64092ce5e08b2bc1b..0df332048424e9ffb8cd476f185d57b740179979 100644 --- a/tensorflow/core/ops/sparse_ops_test.cc +++ b/tensorflow/core/ops/sparse_ops_test.cc @@ -187,8 +187,8 @@ TEST(SparseOpsTest, SparseTensorDenseMatMul_ShapeFn) { // second output dim comes from b, depending on adjoint_b value. INFER_OK(op, "?;?;?;?", "[?,?]"); - INFER_OK(op, "?;?;?;[?,?]", "[?,d3_1]"); // use d3_1, !adjoint_b. - INFER_OK(op, "?;?;?;[1,2]", "[?,d3_1]"); // use d3_1, !adjoint_b. + INFER_OK(op, "?;?;?;[?,?]", "[?,d3_1]"); // use d3_1, !adjoint_b. + INFER_OK(op, "?;?;?;[1,2]", "[?,d3_1]"); // use d3_1, !adjoint_b. INFER_OK(op, "?;?;[2];[1,2]", "[?,d3_1]"); // use d3_1, !adjoint_b. set_adjoints(false, true); diff --git a/tensorflow/core/ops/state_ops.cc b/tensorflow/core/ops/state_ops.cc index b86c0b3990fa55c89e54143a5c4c75958fd8a10f..da5f091e9f1988721b1947ad812851e0322efa9e 100644 --- a/tensorflow/core/ops/state_ops.cc +++ b/tensorflow/core/ops/state_ops.cc @@ -648,4 +648,37 @@ output: A copy of the input before increment. If nothing else modifies the input, the values produced will all be distinct. )doc"); +REGISTER_OP("ResourceCountUpTo") + .Input("resource: resource") + .Output("output: T") + .Attr("limit: int") + .Attr("T: {int32, int64}") + .SetShapeFn([](InferenceContext* c) { + auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data == nullptr || handle_data->empty()) { + return errors::InvalidArgument("Handle has no shape/type information."); + } + shape_inference::ShapeAndType shape_and_type = (*handle_data)[0]; + DataType value_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("T", &value_dtype)); + if (value_dtype != shape_and_type.dtype) { + return errors::InvalidArgument( + "Data types do not match: ", DataTypeString(value_dtype), " and ", + DataTypeString(shape_and_type.dtype)); + } + ShapeHandle output; + TF_RETURN_IF_ERROR(c->WithRank(shape_and_type.shape, 0, &output)); + c->set_output(0, output); + return Status::OK(); + }) + .Doc(R"doc( +Increments variable pointed to by 'resource' until it reaches 'limit'. + +resource: Should be from a scalar `Variable` node. +limit: If incrementing ref would bring it above limit, instead generates an + 'OutOfRange' error. +output: A copy of the input before increment. If nothing else modifies the + input, the values produced will all be distinct. +)doc"); + } // namespace tensorflow diff --git a/tensorflow/core/ops/stateless_random_ops.cc b/tensorflow/core/ops/stateless_random_ops.cc index b222b5b2416dfac09e1dd1abd15862e317b064c7..7c00fdb99fb59a37751c4cb1797f7c51c801d3af 100644 --- a/tensorflow/core/ops/stateless_random_ops.cc +++ b/tensorflow/core/ops/stateless_random_ops.cc @@ -45,7 +45,8 @@ static Status StatelessShape(shape_inference::InferenceContext* context) { .SetShapeFn(StatelessShape) // This op is exposed through contrib/stateless only. The interface may change. -REGISTER_STATELESS_OP("StatelessRandomUniform").Doc(R"doc( +REGISTER_STATELESS_OP("StatelessRandomUniform") + .Doc(R"doc( Outputs deterministic pseudorandom random values from a uniform distribution. The generated values follow a uniform distribution in the range `[0, 1)`. The @@ -60,7 +61,8 @@ output: Random values with specified shape. )doc"); // This op is exposed through contrib/stateless only. The interface may change. -REGISTER_STATELESS_OP("StatelessRandomNormal").Doc(R"doc( +REGISTER_STATELESS_OP("StatelessRandomNormal") + .Doc(R"doc( Outputs deterministic pseudorandom values from a normal distribution. The generated values will have mean 0 and standard deviation 1. @@ -74,7 +76,8 @@ output: Random values with specified shape. )doc"); // This op is exposed through contrib/stateless only. The interface may change. -REGISTER_STATELESS_OP("StatelessTruncatedNormal").Doc(R"doc( +REGISTER_STATELESS_OP("StatelessTruncatedNormal") + .Doc(R"doc( Outputs deterministic pseudorandom values from a truncated normal distribution. The generated values follow a normal distribution with mean 0 and standard diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD index c937fea0490a1e410504f11e50eae60b42eda92a..901fb79d6aa3df8a21df5a4f60f798bd6c00d720 100644 --- a/tensorflow/core/platform/cloud/BUILD +++ b/tensorflow/core/platform/cloud/BUILD @@ -15,7 +15,9 @@ load( filegroup( name = "all_files", srcs = glob( - ["**/*"], + include = [ + "**/*", + ], exclude = [ "**/METADATA", "**/OWNERS", @@ -41,12 +43,8 @@ cc_library( cc_library( name = "gcs_file_system", - srcs = [ - "gcs_file_system.cc", - ], - hdrs = [ - "gcs_file_system.h", - ], + srcs = ["gcs_file_system.cc"], + hdrs = ["gcs_file_system.h"], linkstatic = 1, # Needed since alwayslink is broken in bazel b/27630669 visibility = ["//visibility:public"], deps = [ @@ -106,9 +104,7 @@ cc_library( cc_library( name = "google_auth_provider", - srcs = [ - "google_auth_provider.cc", - ], + srcs = ["google_auth_provider.cc"], hdrs = [ "auth_provider.h", "google_auth_provider.h", @@ -116,7 +112,6 @@ cc_library( visibility = ["//tensorflow:__subpackages__"], deps = [ ":curl_http_request", - ":http_request", ":oauth_client", ":retrying_utils", "//tensorflow/core:lib", diff --git a/tensorflow/core/platform/cloud/curl_http_request.cc b/tensorflow/core/platform/cloud/curl_http_request.cc index e1f8867b38a943132fa18826b1e7afbf31fd2336..e2d935f35eb5134baff6364125df4b8c79205867 100644 --- a/tensorflow/core/platform/cloud/curl_http_request.cc +++ b/tensorflow/core/platform/cloud/curl_http_request.cc @@ -512,8 +512,10 @@ int CurlHttpRequest::ProgressCallback(void* this_object, curl_off_t dltotal, } if (now - that->last_progress_timestamp_ > kInactivityTimeoutSeconds) { - LOG(ERROR) << "The transmission has been stuck at " << current_progress - << " bytes for " << now - that->last_progress_timestamp_ + LOG(ERROR) << "The transmission of request " << this_object + << " has been stuck at " << current_progress << " of " + << dltotal + ultotal << " bytes for " + << now - that->last_progress_timestamp_ << " seconds and will be aborted."; return 1; // Will abort the request. } diff --git a/tensorflow/core/platform/default/build_config.bzl b/tensorflow/core/platform/default/build_config.bzl index 2c14ea917c092a3009cd235b0d9d65cc252b3402..e4518a8e2fdfd5a4a23c86a4b287b6f9c7183ef8 100644 --- a/tensorflow/core/platform/default/build_config.bzl +++ b/tensorflow/core/platform/default/build_config.bzl @@ -467,7 +467,7 @@ def tf_additional_core_deps(): "//conditions:default": [], }) + select({ "//tensorflow:with_s3_support": [ - "//tensorflow/contrib/s3:s3_file_system", + "//tensorflow/core/platform/s3:s3_file_system", ], "//conditions:default": [], }) diff --git a/tensorflow/core/platform/default/gpu_tracer.cc b/tensorflow/core/platform/default/gpu_tracer.cc index 3f855461276c6d7fec03af7cfdcb99f03287c563..e52e37ad7120c70e2319a591eb94999fdabbd6cb 100644 --- a/tensorflow/core/platform/default/gpu_tracer.cc +++ b/tensorflow/core/platform/default/gpu_tracer.cc @@ -315,10 +315,13 @@ class GPUTracerImpl : public GPUTracer, }; return new Impl(name); } - Tracer *StartTracing(StringPiece label) override { + Tracer *StartTracing(StringPiece label, bool is_expensive) override { // We don't do anything with 'TraceMe' regions yet. return nullptr; } + Tracer *StartTracing(StringPiece label) { + return StartTracing(label, /*is_expensive=*/true); + } protected: // This callback is used exclusively by CUPTIManager. diff --git a/tensorflow/core/platform/posix/port.cc b/tensorflow/core/platform/posix/port.cc index 3b17bac80896c6e042af4314b2947d97e45cbdf3..93a59348c8a5be1d7399f35aad8a4468a03d1f2b 100644 --- a/tensorflow/core/platform/posix/port.cc +++ b/tensorflow/core/platform/posix/port.cc @@ -29,7 +29,7 @@ limitations under the License. #include #include #include -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY #include "snappy.h" #endif #if (defined(__APPLE__) && defined(__MACH__)) || defined(__FreeBSD__) @@ -126,7 +126,7 @@ void AdjustFilenameForLogging(string* filename) { } bool Snappy_Compress(const char* input, size_t length, string* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); @@ -139,7 +139,7 @@ bool Snappy_Compress(const char* input, size_t length, string* output) { bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; @@ -147,7 +147,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length, } bool Snappy_Uncompress(const char* input, size_t length, char* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::RawUncompress(input, length, output); #else return false; diff --git a/tensorflow/contrib/s3/BUILD b/tensorflow/core/platform/s3/BUILD similarity index 100% rename from tensorflow/contrib/s3/BUILD rename to tensorflow/core/platform/s3/BUILD diff --git a/tensorflow/contrib/s3/s3_crypto.cc b/tensorflow/core/platform/s3/s3_crypto.cc similarity index 96% rename from tensorflow/contrib/s3/s3_crypto.cc rename to tensorflow/core/platform/s3/s3_crypto.cc index 1450384dc0f8b4d4f30c8776f6c1e31b0affeea7..d7062a59d2c88195b67cdf3c62cb14164e1038f0 100644 --- a/tensorflow/contrib/s3/s3_crypto.cc +++ b/tensorflow/core/platform/s3/s3_crypto.cc @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/s3/s3_crypto.h" +#include "tensorflow/core/platform/s3/s3_crypto.h" #include #include @@ -71,7 +71,7 @@ class S3Sha256OpenSSLImpl : public Aws::Utils::Crypto::Hash { SHA256_Init(&sha256); auto currentPos = stream.tellg(); - if (currentPos == -1) { + if (currentPos == std::streampos(std::streamoff(-1))) { currentPos = 0; stream.clear(); } diff --git a/tensorflow/contrib/s3/s3_crypto.h b/tensorflow/core/platform/s3/s3_crypto.h similarity index 100% rename from tensorflow/contrib/s3/s3_crypto.h rename to tensorflow/core/platform/s3/s3_crypto.h diff --git a/tensorflow/contrib/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc similarity index 99% rename from tensorflow/contrib/s3/s3_file_system.cc rename to tensorflow/core/platform/s3/s3_file_system.cc index daced83145353c52ae19e2b7e8491b5fcb31cc1f..51c85592bf43bdfb68c4ba90d19d28582560d6d4 100644 --- a/tensorflow/contrib/s3/s3_file_system.cc +++ b/tensorflow/core/platform/s3/s3_file_system.cc @@ -12,10 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/s3/s3_file_system.h" -#include "tensorflow/contrib/s3/s3_crypto.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/s3/s3_file_system.h" +#include "tensorflow/core/platform/s3/s3_crypto.h" #include #include diff --git a/tensorflow/contrib/s3/s3_file_system.h b/tensorflow/core/platform/s3/s3_file_system.h similarity index 100% rename from tensorflow/contrib/s3/s3_file_system.h rename to tensorflow/core/platform/s3/s3_file_system.h diff --git a/tensorflow/contrib/s3/s3_file_system_test.cc b/tensorflow/core/platform/s3/s3_file_system_test.cc similarity index 99% rename from tensorflow/contrib/s3/s3_file_system_test.cc rename to tensorflow/core/platform/s3/s3_file_system_test.cc index 949281fad4a6c6d67f12d4de4e6be0e5e4d025ea..0b42f5fcec0041a01a571b1e38dedaa7ef191c22 100644 --- a/tensorflow/contrib/s3/s3_file_system_test.cc +++ b/tensorflow/core/platform/s3/s3_file_system_test.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/contrib/s3/s3_file_system.h" +#include "tensorflow/core/platform/s3/s3_file_system.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/gtl/stl_util.h" diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h index b7724bbeae4d470c7374483264b3ba25d5ed0190..bb8e902efc25420ce1b7beb00a1911500c627a00 100644 --- a/tensorflow/core/platform/tracing.h +++ b/tensorflow/core/platform/tracing.h @@ -169,10 +169,10 @@ class Tracing::Engine { // Start tracing under the specified label. Caller should delete the result // to stop tracing. // May return nullptr if tracing is not supported. - virtual Tracer* StartTracing(StringPiece label) = 0; + virtual Tracer* StartTracing(StringPiece label, bool is_expensive) = 0; // Same as above, but implementations can avoid copying the string. - virtual Tracer* StartTracing(string&& label) { - return StartTracing(StringPiece(label)); + virtual Tracer* StartTracing(string&& label, bool is_expensive) { + return StartTracing(StringPiece(label), is_expensive); } }; @@ -218,12 +218,14 @@ class Tracing::ScopedAnnotation { class Tracing::TraceMe { public: explicit TraceMe(StringPiece name); + TraceMe(StringPiece name, bool is_expensive); // If tracing is enabled, set up a traceMe with a label of // ":". This can be cheaper than the // single-argument constructor because the concatenation of the // label string is only done if tracing is enabled. TraceMe(StringPiece name_part1, StringPiece name_part2); + TraceMe(StringPiece name_part1, StringPiece name_part2, bool is_expensive); private: std::unique_ptr tracer_; @@ -245,19 +247,24 @@ inline Tracing::ScopedAnnotation::ScopedAnnotation(StringPiece name_part1, } } -inline Tracing::TraceMe::TraceMe(StringPiece name) { +inline Tracing::TraceMe::TraceMe(StringPiece name) : TraceMe(name, true) {} + +inline Tracing::TraceMe::TraceMe(StringPiece name, bool is_expensive) { auto e = Tracing::engine(); if (e && e->IsEnabled()) { - tracer_.reset(e->StartTracing(name)); + tracer_.reset(e->StartTracing(name, is_expensive)); } } -inline Tracing::TraceMe::TraceMe(StringPiece name_part1, - StringPiece name_part2) { +inline Tracing::TraceMe::TraceMe(StringPiece name_part1, StringPiece name_part2) + : TraceMe(name_part1, name_part2, true) {} + +inline Tracing::TraceMe::TraceMe(StringPiece name_part1, StringPiece name_part2, + bool is_expensive) { auto e = Tracing::engine(); if (e && e->IsEnabled()) { - tracer_.reset( - e->StartTracing(strings::StrCat(name_part1, ":", name_part2))); + tracer_.reset(e->StartTracing(strings::StrCat(name_part1, ":", name_part2), + is_expensive)); } } diff --git a/tensorflow/core/platform/windows/port.cc b/tensorflow/core/platform/windows/port.cc index 85b53e07c439e02f63d4600c57c925f3b8d843b9..e327d53949caf7e2d30e6deba0be2848f010afc2 100644 --- a/tensorflow/core/platform/windows/port.cc +++ b/tensorflow/core/platform/windows/port.cc @@ -20,7 +20,7 @@ limitations under the License. #include #include #include -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY #include "snappy.h" #endif @@ -118,7 +118,7 @@ void AdjustFilenameForLogging(string* filename) { } bool Snappy_Compress(const char* input, size_t length, string* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY output->resize(snappy::MaxCompressedLength(length)); size_t outlen; snappy::RawCompress(input, length, &(*output)[0], &outlen); @@ -131,7 +131,7 @@ bool Snappy_Compress(const char* input, size_t length, string* output) { bool Snappy_GetUncompressedLength(const char* input, size_t length, size_t* result) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::GetUncompressedLength(input, length, result); #else return false; @@ -139,7 +139,7 @@ bool Snappy_GetUncompressedLength(const char* input, size_t length, } bool Snappy_Uncompress(const char* input, size_t length, char* output) { -#ifdef SNAPPY +#ifdef TF_USE_SNAPPY return snappy::RawUncompress(input, length, output); #else return false; diff --git a/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc b/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc index c39d44b7fa4285b890bcfc60372547977107daa5..d05143aff9b8cc0b9a0e9af9445ba79345e4bf62 100644 --- a/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc +++ b/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc @@ -48,7 +48,7 @@ class TFProfAdvisorTest : public ::testing::Test { for (const auto& attr : attrs) { (*def->mutable_attr())[attr.first].set_s(attr.second); } - std::unique_ptr node(new TFGraphNode(def, -1)); + std::unique_ptr node(new TFGraphNode(def, -1, nullptr)); NodeExecStats node_stat; node_stat.set_all_start_micros(start_miros); diff --git a/tensorflow/core/profiler/internal/print_model_analysis.cc b/tensorflow/core/profiler/internal/print_model_analysis.cc index 575ae182ee8f135d8df727bc7e6eaa43aba592c8..7a0d590262fe623f701e21c979e53f2abc103305 100644 --- a/tensorflow/core/profiler/internal/print_model_analysis.cc +++ b/tensorflow/core/profiler/internal/print_model_analysis.cc @@ -119,8 +119,8 @@ void DeleteProfiler() { } } -void AddStep(int64 step, const string* graph, const string* run_meta, - const string* op_log) { +double AddStep(int64 step, const string* graph, const string* run_meta, + const string* op_log) { CHECK(tf_stat); CHECK(graph && !graph->empty()); @@ -144,6 +144,7 @@ void AddStep(int64 step, const string* graph, const string* run_meta, op_log_ptr->ParseFromString(*op_log); tf_stat->AddOpLogProto(std::move(op_log_ptr)); } + return tf_stat->run_coverage(); } string Profile(const string* command, const string* options) { @@ -154,6 +155,7 @@ string Profile(const string* command, const string* options) { } void WriteProfile(const string* filename) { + CHECK(tf_stat); CHECK(filename) << "empty file name when asking to write profile."; tf_stat->WriteProfile(*filename); } diff --git a/tensorflow/core/profiler/internal/print_model_analysis.h b/tensorflow/core/profiler/internal/print_model_analysis.h index e4d01041a84e31c9554a9e5031aab40c4fd68234..31ff5b07b060b43fab6c0b458f6f43c4dcc0576b 100644 --- a/tensorflow/core/profiler/internal/print_model_analysis.h +++ b/tensorflow/core/profiler/internal/print_model_analysis.h @@ -35,8 +35,8 @@ bool NewProfiler(const string* graph, const string* op_log); void DeleteProfiler(); -void AddStep(int64 step, const string* graph, const string* run_meta, - const string* op_log); +double AddStep(int64 step, const string* graph, const string* run_meta, + const string* op_log); // Write the profiler's profile to a proto buffer. void WriteProfile(const string* filename); diff --git a/tensorflow/core/profiler/internal/tfprof_graph.cc b/tensorflow/core/profiler/internal/tfprof_graph.cc index 3766365bf852da6e2e13a9acbff68d4130b39e56..db7ae3b39715d0a8ac20644907fc4addd43c86ce 100644 --- a/tensorflow/core/profiler/internal/tfprof_graph.cc +++ b/tensorflow/core/profiler/internal/tfprof_graph.cc @@ -31,7 +31,7 @@ GraphNode* TFGraph::CreateParentNode(const string& name) { node_defs_.back()->set_name(name); node_defs_.back()->set_op(kTFGraphParent); parent_nodes_[name] = std::unique_ptr( - new TFGraphNode(node_defs_.back().get(), -1)); + new TFGraphNode(node_defs_.back().get(), -1, nullptr)); nodes_map_[name] = std::unique_ptr(new GraphNode(parent_nodes_[name].get())); return nodes_map_[name].get(); diff --git a/tensorflow/core/profiler/internal/tfprof_node.cc b/tensorflow/core/profiler/internal/tfprof_node.cc index f283fafc0fa4446ec2bdeec1cf36e75f05ecbf90..671b65d708f57713d984331de73ddf305675b792 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.cc +++ b/tensorflow/core/profiler/internal/tfprof_node.cc @@ -19,19 +19,15 @@ limitations under the License. namespace tensorflow { namespace tfprof { -namespace { bool CountAsAcceleratorTime(const string& device) { return device.find("stream:all") != device.npos; } - bool CountAsCPUTime(const string& device) { return RE2::FullMatch(device, ".*/(device:gpu|gpu|device:cpu|cpu|device:sycl):\\d+"); } - bool IsCanonicalDevice(const string& device) { return CountAsCPUTime(device); } -} // namespace // Notes about start and end time from the NodeExecStats proto: // For GPU, there is no difference between op_end_rel_micros and // all_end_rel_micros. All are kernel times. @@ -89,16 +85,28 @@ void ExecStep::AddMemoryStats(const string& dev, } exec_.set_memory_intialized(true); + int accelerator_allocator_cnt = 0; for (const auto& mem : step_stat.memory()) { // TODO(xpan): Fix this hack. Currently the allocator name seems quite // ad-hoc. if (mem.allocator_name().find("GPU") == mem.allocator_name().npos) { continue; } + ++accelerator_allocator_cnt; exec_.set_allocator_bytes_in_use( std::max(static_cast(exec_.allocator_bytes_in_use()), static_cast(mem.allocator_bytes_in_use()))); + Allocation allocation; + for (const auto& alloc : mem.allocation_records()) { + allocation.add_allocation_records()->MergeFrom(alloc); + } + allocations_.push_back(allocation); } + if (accelerator_allocator_cnt > 1) { + fprintf(stderr, "found %d gpu allocator for 1 node\n", + accelerator_allocator_cnt); + } + int64 total_output_bytes = 0; for (const auto& output : step_stat.output()) { if (output.has_tensor_description() && diff --git a/tensorflow/core/profiler/internal/tfprof_node.h b/tensorflow/core/profiler/internal/tfprof_node.h index 34bc0a581d30c11a9d95832279b02f1383c920a7..e2d0563a0747d7bec74ce3aeb9d5995f47cff915 100644 --- a/tensorflow/core/profiler/internal/tfprof_node.h +++ b/tensorflow/core/profiler/internal/tfprof_node.h @@ -105,8 +105,22 @@ class ExecStep { const { return op_execs_; } + const std::map>>& cpu_execs() + const { + return cpu_execs_; + } + int64 all_start_micros() const { return exec_.all_start_micros(); } int64 latest_end_micros() const { return exec_.latest_end_micros(); } + int64 lastest_schedule_end_micros() const { + int64 ret = 0; + for (const auto& exec : cpu_execs_) { + for (const auto& pair : exec.second) { + ret = std::max(ret, pair.first + pair.second); + } + } + return ret; + } int64 requested_bytes() const { return exec_.requested_bytes(); } int64 peak_bytes() const { return exec_.peak_bytes(); } @@ -127,6 +141,8 @@ class ExecStep { return exec_.allocator_bytes_in_use(); } + const std::vector& allocations() const { return allocations_; } + const ExecProfile& ToProto() { exec_.mutable_accelerator_execs()->clear(); for (const auto& e : accelerator_execs_) { @@ -161,6 +177,11 @@ class ExecStep { mem_pb.set_ptr(mem.second.second); } + exec_.mutable_allocations()->Clear(); + for (const auto& r : allocations_) { + exec_.add_allocations()->MergeFrom(r); + } + return exec_; } @@ -175,6 +196,8 @@ class ExecStep { cpu_execs_.clear(); op_execs_.clear(); + allocations_.clear(); + for (const auto& exec_time : exec_.accelerator_execs()) { auto& exec = accelerator_execs_[exec_time.first]; auto& op_exec = op_execs_[exec_time.first]; @@ -196,6 +219,10 @@ class ExecStep { mem.first = output_mem.second.bytes(); mem.second = output_mem.second.ptr(); } + + for (const auto& r : exec_.allocations()) { + allocations_.push_back(r); + } } private: @@ -215,6 +242,9 @@ class ExecStep { std::set devices_; // output_idx -> {output_bytes, memory_ptr} std::map> output_memory_; + + // The history of accelerator allocations and deallocations of this step. + std::vector allocations_; }; #define GRAPH_NODE_BYTES(type) \ @@ -238,11 +268,15 @@ class ExecStep { class TFGraphNode { public: TFGraphNode(const ProfileNode& node, const ProfileProto& profile, - const std::map* id_to_string) { + const std::map* id_to_string, + const std::map>* nodes_map) { + nodes_map_ = nodes_map; FromProto(node, profile, id_to_string); } - TFGraphNode(const NodeDef* node, int64 id) { + TFGraphNode(const NodeDef* node, int64 id, + const std::map>* nodes_map) { + nodes_map_ = nodes_map; node_.set_id(id); node_.set_name(node->name()); node_.set_op(node->op()); @@ -269,17 +303,9 @@ class TFGraphNode { op_types_.insert(node->op()); } - void AddInput(TFGraphNode* input, int32 output_idx, int input_idx) { - src_output_idx_[input->name()] = output_idx; - - inputs_[input_idx] = input->name(); - const auto& output_shape = input->output_shapes().find(output_idx); - // Always create an empty vec even if the shape info might be missing. - std::vector& shape_vec = input_shapes_[input_idx]; - if (output_shape != input->output_shapes().end()) { - shape_vec.assign(output_shape->second.begin(), - output_shape->second.end()); - } + void AddInput(const string& input, int64 output_index, int input_idx) { + inputs_[input_idx] = input; + src_output_idx_[input] = output_index; } void AddOpType(const string& op_type) { op_types_.insert(op_type); } @@ -416,9 +442,6 @@ class TFGraphNode { } const std::map& inputs() const { return inputs_; } - const std::map& src_output_idx() const { - return src_output_idx_; - } // Number of times the graph node is executed. When step < 0, the // average number of times executed across all steps. @@ -526,14 +549,30 @@ class TFGraphNode { return exec->second.latest_end_micros(); } + int64 lastest_schedule_end_micros(int64 step) const { + auto exec = execs_.find(step); + if (exec == execs_.end()) { + return 0; + } + return exec->second.lastest_schedule_end_micros(); + } + const std::map>>& op_execs( int64 step) const { auto exec = execs_.find(step); if (exec == execs_.end()) { - return empty_op_execs_; + return empty_execs_; } return exec->second.op_execs(); } + const std::map>>& cpu_execs( + int64 step) const { + auto exec = execs_.find(step); + if (exec == execs_.end()) { + return empty_execs_; + } + return exec->second.cpu_execs(); + } const std::map& all_op_execs() const { return execs_; } @@ -551,12 +590,12 @@ class TFGraphNode { } return exec->second.host_temp_bytes(); } - int64 accelerator_persistent_bytes(int64 step) const { - auto exec = execs_.find(step); - if (exec == execs_.end()) { - return 0; + int64 accelerator_persistent_bytes() const { + int64 persistent_bytes = 0; + for (const auto& exec : execs_) { + persistent_bytes += exec.second.accelerator_persistent_bytes(); } - return exec->second.accelerator_persistent_bytes(); + return persistent_bytes; } int64 host_persistent_bytes(int64 step) const { auto exec = execs_.find(step); @@ -581,6 +620,14 @@ class TFGraphNode { return exec->second.allocator_bytes_in_use(); } + const std::vector& allocations(int64 step) const { + auto exec = execs_.find(step); + if (exec == execs_.end()) { + return empty_allocations_; + } + return exec->second.allocations(); + } + int64 parameters() const { if (!shape().empty()) { int64 params = 1; @@ -628,18 +675,44 @@ class TFGraphNode { const std::map>& output_shapes() const { return output_shapes_; } - const std::map>& input_shapes() const { - return input_shapes_; + + const std::map> input_shapes() const { + std::map> input_shapes; + for (const auto& inp : inputs_) { + // Always create an empty vec even if the shape info might be missing. + std::vector& shape_vec = input_shapes[inp.first]; + if (!nodes_map_) continue; + auto input_it = nodes_map_->find(inp.second); + if (input_it == nodes_map_->end()) continue; + auto output_it = src_output_idx_.find(inp.second); + if (output_it == src_output_idx_.end()) continue; + + const TFGraphNode* input_node = input_it->second.get(); + if (!input_node) continue; + const auto& output_shapes = input_node->output_shapes(); + const auto& output_shape = output_shapes.find(output_it->second); + if (output_shape == output_shapes.end()) continue; + + if (output_shape != input_node->output_shapes().end()) { + shape_vec.assign(output_shape->second.begin(), + output_shape->second.end()); + } + } + return input_shapes; } private: + // maps graph node name to TFGraphNode. Not owned. + const std::map>* nodes_map_; + // inputs to the node. input index -> input node name. std::map inputs_; + // The output index of the source node. std::map src_output_idx_; - + // proto for serialize/deserialized representation of the node. ProfileNode node_; - + // Python call stack that creates the name. std::unique_ptr call_stack_; - + // Shape of the node (e.g. Variable) if available. std::vector shape_; // Won't missing input_idx. But some shapes might be empty (unknown). std::map> input_shapes_; @@ -651,8 +724,10 @@ class TFGraphNode { std::map execs_; + // Placeholder for empty cases. std::map> empty_output_memory_; - std::map>> empty_op_execs_; + std::map>> empty_execs_; + std::vector empty_allocations_; }; class TFMultiGraphNode { @@ -806,6 +881,10 @@ class TFMultiGraphNode { }; bool IsPlacedOnAccelerator(const string& device); +bool CountAsAcceleratorTime(const string& device); +bool CountAsCPUTime(const string& device); +bool IsCanonicalDevice(const string& device); + } // namespace tfprof } // namespace tensorflow diff --git a/tensorflow/core/profiler/internal/tfprof_scope.cc b/tensorflow/core/profiler/internal/tfprof_scope.cc index 128b296d5c32e0bf8d87dfc794f743c525dba69b..988bed71cc8262c70445c06ca0c2b9b34145d9ce 100644 --- a/tensorflow/core/profiler/internal/tfprof_scope.cc +++ b/tensorflow/core/profiler/internal/tfprof_scope.cc @@ -35,7 +35,7 @@ ScopeNode* TFScope::CreateParentNode(const string& name) { node_defs_.back()->set_name(name); node_defs_.back()->set_op(kTFScopeParent); parent_nodes_[name] = std::unique_ptr( - new TFGraphNode(node_defs_.back().get(), -1)); + new TFGraphNode(node_defs_.back().get(), -1, nullptr)); nodes_map_[name] = std::unique_ptr(new ScopeNode(parent_nodes_[name].get())); return nodes_map_[name].get(); diff --git a/tensorflow/core/profiler/internal/tfprof_stats.cc b/tensorflow/core/profiler/internal/tfprof_stats.cc index b4b98141f3f1707b6107d1500431e781ae7140b4..7943c075e0243e652cb19125dae95b04dc709f97 100644 --- a/tensorflow/core/profiler/internal/tfprof_stats.cc +++ b/tensorflow/core/profiler/internal/tfprof_stats.cc @@ -36,7 +36,9 @@ bool CreateRunMetadataNode(const string& name, NodeDef* def) { } def->set_name(name); // TODO(xpan): Better operation type. - def->set_op("RunTimeOp"); + // This is because some times a node doesn't have a op type, + // so we use node name as the op type. + def->set_op(name); return true; } } // namespace @@ -86,7 +88,7 @@ TFStats::TFStats(const string& filename, } for (const auto& node_pb : profile.nodes()) { std::unique_ptr node( - new TFGraphNode(node_pb.second, profile, &id_to_string_)); + new TFGraphNode(node_pb.second, profile, &id_to_string_, &nodes_map_)); nodes_map_.insert(std::pair>( node_pb.second.name(), std::move(node))); } @@ -178,12 +180,14 @@ const MultiGraphNodeProto& TFStats::ShowMultiGraphNode( void TFStats::AddGraph(std::unique_ptr graph) { std::map node_defs; + bool node_added = false; for (const NodeDef& node : graph->node()) { if (nodes_map_.find(node.name()) != nodes_map_.end()) { continue; } - nodes_map_[node.name()] = - std::unique_ptr(new TFGraphNode(&node, nodes_map_.size())); + node_added = true; + nodes_map_[node.name()] = std::unique_ptr( + new TFGraphNode(&node, nodes_map_.size(), &nodes_map_)); node_defs[node.name()] = &node; } for (auto it = node_defs.begin(); it != node_defs.end(); it++) { @@ -192,6 +196,7 @@ void TFStats::AddGraph(std::unique_ptr graph) { string node_input = it->second->input(i); int output_idx = 0; // input name format can be: "^node:src_output" + // if not :src_output, then it's the first one (further verify?) auto prefix_pos = node_input.find(":"); if (prefix_pos != node_input.npos) { std::vector input_parts = str_util::Split(node_input, ":"); @@ -204,15 +209,18 @@ void TFStats::AddGraph(std::unique_ptr graph) { if (node_input.substr(0, 1) == "^") { node_input = node_input.substr(1); } - auto input_node = nodes_map_.find(node_input); - // TODO(xpan): P1: Add the input even if it doesn't exist yet, because - // this can be a partial graph. - if (input_node == nodes_map_.end()) { - continue; - } - node->AddInput(input_node->second.get(), output_idx, i); + // Delay input TFGraphNode retrieval as late as possible. + // In long run, when we have TensorFlow runtime graph, the + // graph connection should be dynamic and per-step. + node->AddInput(node_input, output_idx, i); } } + if (node_added) { + graph_view_.reset(nullptr); + scope_view_.reset(nullptr); + op_view_.reset(nullptr); + code_view_.reset(nullptr); + } } void TFStats::AddOpLogProto(std::unique_ptr op_log) { @@ -263,10 +271,11 @@ void TFStats::AddRunMeta(int64 step, std::unique_ptr run_meta) { NodeDef def; if (CreateRunMetadataNode(name, &def)) { nodes_map_[name] = std::unique_ptr( - new TFGraphNode(&def, nodes_map_.size())); + new TFGraphNode(&def, nodes_map_.size(), &nodes_map_)); nodes_map_.at(name)->AddStepStat(step, dev_stat.device(), node_stat); } } else { + covered_nodes_.insert(node->second->id()); node->second->AddStepStat(step, dev_stat.device(), node_stat); } } diff --git a/tensorflow/core/profiler/internal/tfprof_stats.h b/tensorflow/core/profiler/internal/tfprof_stats.h index bb4baea738ea0ed46b8475f81045451875dec99c..d46d9235560c673323d243a40f21bbd06aa9416d 100644 --- a/tensorflow/core/profiler/internal/tfprof_stats.h +++ b/tensorflow/core/profiler/internal/tfprof_stats.h @@ -66,6 +66,9 @@ class TFStats { } const std::set& steps() const { return steps_; } bool has_code_traces() const { return has_code_traces_; } + double run_coverage() const { + return covered_nodes_.size() / (nodes_map_.size() + 1e-10); + } void BuildView(const string& cmd); void BuildAllViews(); @@ -104,13 +107,16 @@ class TFStats { std::unique_ptr code_view_; std::unique_ptr op_view_; std::unique_ptr ckpt_reader_; - // Store TFGraphNode instead of TFGraphNode* to avoid large number of - // dynamic alloc. + // TODO(xpan): Store TFGraphNode instead of TFGraphNode* to avoid large + // number of dynamic alloc. + // Maps from graph node name to TFGraphNode. std::map> nodes_map_; GraphNodeProto empty_graph_node_; MultiGraphNodeProto empty_multi_graph_node_; std::map id_to_string_; + // Graph nodes covered by RunMetdata, that is traced with run time stats. + std::set covered_nodes_; }; } // namespace tfprof diff --git a/tensorflow/core/profiler/internal/tfprof_timeline.cc b/tensorflow/core/profiler/internal/tfprof_timeline.cc index 1732574cc41f48a3422d62eb11ad28baac2288a2..bdb000747db72900d748c22140ca38e571db6691 100644 --- a/tensorflow/core/profiler/internal/tfprof_timeline.cc +++ b/tensorflow/core/profiler/internal/tfprof_timeline.cc @@ -25,6 +25,8 @@ limitations under the License. namespace tensorflow { namespace tfprof { namespace { +int kMaxDisplayedMemNode = 10; + string GetTimeDevName(const string& dev) { if (dev.find("stream") != dev.npos) { return strings::StrCat("Op execution threads: ", dev); @@ -85,14 +87,41 @@ void ChromeTraceFormatter::EmitFlowEnd(const string& name, int64 ts, int64 pid, events_.push_back(event); } -void ChromeTraceFormatter::EmitCounter(const string& category, - const string& name, int64 pid, int64 ts, - const string& device, int64 bytes) { - Json::Value event = CreateEvent("C", category, name, pid, 0, ts); +void ChromeTraceFormatter::EmitCounter( + const string& category, const string& name, int64 pid, int64 ts, + const string& device, int64 bytes, + const std::map>& tensor_mem) { + Json::Value event = CreateEvent("C", category, "Allocated Bytes", pid, 0, ts); Json::Value args(Json::objectValue); - args[device] = Json::Value(bytes); + args["Allocator Bytes in Use"] = Json::Value(bytes); event["args"] = args; events_.push_back(event); + + // TODO(xpan): chrome://tracing is not ideal visualization for memory. + // It would be great to have a customized UI for it. + Json::Value event2 = + CreateEvent("C", category, "Top Allocations", pid + 1, 0, ts); + Json::Value args2(Json::objectValue); + // Need to reserve the same args for all locations. + for (int i = 1; i < kMaxDisplayedMemNode; ++i) { + args2[strings::Printf("Top Allocation %02d", i)] = Json::Value("N/A"); + } + int count = 0; + for (auto it = tensor_mem.rbegin(); it != tensor_mem.rend(); ++it) { + for (const string& t : it->second) { + if (bytes < it->first || count >= kMaxDisplayedMemNode) { + break; + } + args2[strings::Printf("Top Allocation %02d", count)] = + Json::Value(strings::StrCat(it->first / 1000000.0, " MB from ", t)); + ++count; + bytes -= it->first; + } + } + args2[strings::StrCat("Not Displayed")] = + Json::Value(strings::Printf("%.2f MB", bytes / 1000000.0)); + event2["args"] = args2; + events_.push_back(event2); } string ChromeTraceFormatter::Format() { @@ -119,71 +148,28 @@ void MemoryTracker::TrackNode(int64 step, const GraphNode* node) { if (!node->Trackable(step)) { return; } + Device& dev = devices_[node->node->canonical_device()]; - int64 end_micros = node->node->latest_end_micros(step); - if (node->node->accelerator_persistent_bytes(step) != 0) { - string tensor_name = strings::StrCat(node->name(), ":", -1); - dev.earliest_ref[tensor_name] = node->node->all_start_micros(step); - dev.tensor_size[tensor_name] = - node->node->accelerator_persistent_bytes(step); - // TODO(xpan): Need latest_ref? - } - if (node->node->accelerator_temp_bytes(step)) { - string tensor_name = strings::StrCat(node->name(), ":", -2); - dev.earliest_ref[tensor_name] = node->node->all_start_micros(step); - dev.latest_ref[tensor_name] = end_micros; - dev.tensor_size[tensor_name] = node->node->accelerator_temp_bytes(step); - } - if (node->node->allocator_bytes_in_use(step) > 0) { - dev.allocator_stats[end_micros] = node->node->allocator_bytes_in_use(step); - } -} -void MemoryTracker::TrackNodeConnection(int64 step, const GraphNode* node, - const GraphNode* src) { - if (!node->Trackable(step) || !src->Trackable(step)) { - return; - } - const auto& output_idx = node->node->src_output_idx().find(src->name()); - if (output_idx == node->node->src_output_idx().end()) { - return; - } - const auto& output = src->node->output_memory(step).find(output_idx->second); - if (output == src->node->output_memory(step).end()) { - return; + std::map allocs; + for (const auto& alloc : node->node->allocations(step)) { + for (const auto& r : alloc.allocation_records()) { + allocs[r.alloc_micros()] += r.alloc_bytes(); + dev.tracked_allocations[r.alloc_micros()] += r.alloc_bytes(); + } } - int64 output_bytes = output->second.first; - uint64 output_ptr = output->second.second; - - Device& src_dev = devices_[src->node->canonical_device()]; - string tensor_name = strings::StrCat(output_ptr); - if (output_ptr == 0) { - fprintf(stderr, "output no ptr\n"); - tensor_name = strings::StrCat(src->node->name(), ":", output_idx->second); + dev.tracked_allocations[0] += node->node->accelerator_persistent_bytes(); + allocs[0] += node->node->accelerator_persistent_bytes(); + + int64 last = 0; + std::map& aggregate_allocs = dev.tensor_allocs[node->name()]; + for (auto it = allocs.begin(); it != allocs.end(); ++it) { + last += it->second; + aggregate_allocs[it->first] = last; } - - src_dev.tensor_size[tensor_name] = output_bytes; - src_dev.earliest_ref[tensor_name] = src->node->all_start_micros(step); - - int64 src_end_micros = src->node->latest_end_micros(step); - - if (src->node->canonical_device() != node->node->canonical_device()) { - int64 transfer_micros = - (src_end_micros + node->node->all_start_micros(step)) / 2; - src_dev.latest_ref[tensor_name] = - std::max(src_dev.latest_ref[tensor_name], transfer_micros); - - Device& dest_dev = devices_[node->node->canonical_device()]; - string dest_tensor_name = - strings::StrCat(tensor_name, node->node->canonical_device()); - dest_dev.tensor_size[dest_tensor_name] = output_bytes; - dest_dev.earliest_ref[dest_tensor_name] = transfer_micros; - dest_dev.latest_ref[dest_tensor_name] = - std::max(dest_dev.latest_ref[dest_tensor_name], - node->node->latest_end_micros(step)); - } else { - src_dev.latest_ref[tensor_name] = std::max( - src_dev.latest_ref[tensor_name], node->node->latest_end_micros(step)); + int64 end_micros = node->node->lastest_schedule_end_micros(step); + if (end_micros > 0 && node->node->allocator_bytes_in_use(step) > 0) { + dev.allocations[end_micros] = node->node->allocator_bytes_in_use(step); } } @@ -222,22 +208,24 @@ void Timeline::GenerateGraphTimeline(const std::vector& gnodes) { for (GraphNode* gnode : gnodes) { AllocateTimeNodes(gnode); } + // To save memory, we only track cross-device (canonical device) flows. for (auto& process : tnodes_) { + if (!IsCanonicalDevice(process.first)) continue; for (auto& tn : process.second) { TimeNode* tnode = tn.second.get(); for (GraphNode* inp : tnode->node->children) { if (!inp->account || !inp->Trackable(step_)) { continue; } - TrackNodeConnection(tnode->node, inp); - for (const auto& kernel_execs : inp->node->op_execs(step_)) { - if (process.first == kernel_execs.first) { - // Not interested in flow withthin the same device. + for (const auto& execs : inp->node->cpu_execs(step_)) { + if (!IsCanonicalDevice(execs.first)) continue; + if (process.first == execs.first) { + // Not interested in flow within the same device. continue; } - for (const auto& exec : kernel_execs.second) { + for (const auto& exec : execs.second) { int64 start_micros = exec.first; - auto cprocess = tnodes_.find(kernel_execs.first); + auto cprocess = tnodes_.find(execs.first); if (cprocess == tnodes_.end()) continue; auto ctn = cprocess->second.find(start_micros); if (ctn == cprocess->second.end()) continue; @@ -258,7 +246,6 @@ void Timeline::GenerateGraphTimeline(const std::vector& gnodes) { Json::Value args(Json::objectValue); args["name"] = Json::Value(tnode->name()); - args["op"] = Json::Value(tnode->name()); chrome_formatter_.EmitRegion(node.first, tnode->exec_micros, process.first, lane.first, "Op", tnode->name(), args); @@ -280,12 +267,40 @@ void Timeline::GenerateGraphTimeline(const std::vector& gnodes) { for (const auto& dev : mem_tracker_.devices()) { int64 pid = AllocatePID(); chrome_formatter_.EmitPID(GetMemoryLaneName(dev.first), pid); + int64 pid2 = AllocatePID(); + chrome_formatter_.EmitPID(GetMemoryLaneName(dev.first) + " allocations", + pid2); + const MemoryTracker::Device& device = dev.second; - for (const auto& alloc_stats : device.allocator_stats) { - chrome_formatter_.EmitCounter("Memory", "Memory Series", pid, - alloc_stats.first, dev.first, - alloc_stats.second); + int64 max_bytes_in_use = 0; + int64 cur_bytes_in_use = 0; + int64 last_point = 0; + for (const auto& alloc : device.allocations) { + cur_bytes_in_use = alloc.second; + max_bytes_in_use = std::max(max_bytes_in_use, cur_bytes_in_use); + // Do not plot too dense to reduce file size. + int64 ts = alloc.first; + if (ts - last_point < 100) continue; + last_point = ts; + + std::map> tensor_mem; + for (const auto& tensor_alloc_it : dev.second.tensor_allocs) { + const auto& tensor_alloc = tensor_alloc_it.second; + auto it = tensor_alloc.lower_bound(ts); + if (it != tensor_alloc.begin()) { + --it; + } + if (it->second > 0) { + tensor_mem[it->second].push_back(tensor_alloc_it.first); + } + } + chrome_formatter_.EmitCounter("Memory", "Memory Series", pid, ts, + dev.first, cur_bytes_in_use, tensor_mem); + } + if (IsPlacedOnAccelerator(dev.first)) { + fprintf(stdout, "%s peak memory: %.2f MB\n", dev.first.c_str(), + max_bytes_in_use / 1000000.0); } } OutputTimeline(); diff --git a/tensorflow/core/profiler/internal/tfprof_timeline.h b/tensorflow/core/profiler/internal/tfprof_timeline.h index 6c62d1046faa58cac2f3371b353671f7ac45ed17..b8174cdecbd764ff784049e75d0a62c038c05978 100644 --- a/tensorflow/core/profiler/internal/tfprof_timeline.h +++ b/tensorflow/core/profiler/internal/tfprof_timeline.h @@ -28,10 +28,12 @@ namespace tfprof { typedef std::map Event; +// Class for generating timeline json output. class ChromeTraceFormatter { public: ChromeTraceFormatter() {} - + // The following methods creates timeline nodes. See chrome tracing format + // document for details. Json::Value CreateEvent(const string& ph, const string& category, const string& name, int64 pid, int64 tid, int64 ts); @@ -47,22 +49,27 @@ class ChromeTraceFormatter { int64 flow_id); void EmitCounter(const string& category, const string& name, int64 pid, - int64 ts, const string& device, int64 bytes); + int64 ts, const string& device, int64 bytes, + const std::map>& tensor_mem); string Format(); private: + // A event is a visualization unit in timeline. std::vector events_; std::vector metadata_; }; +// A process (time series of events) in the timeline. class Process { public: Process(const string& device, int64 pid) : device(device), pid(pid) {} // Each lane is a map from start_time to end_time. std::vector> lanes; + // device for the time series. string device; + // unique id for the time series. int64 pid; }; @@ -96,19 +103,16 @@ class MemoryTracker { public: class Device { public: - // The first 3 fields are predicted. - std::map tensor_size; - std::map earliest_ref; - std::map latest_ref; + // map from tensor name to a pair of . + std::map> tensor_allocs; // ground truth memory stats. time->bytes. - std::map allocator_stats; + std::map allocations; + // tracked allocations, might miss some bytes. + std::map tracked_allocations; }; void TrackNode(int64 step, const GraphNode* node); - void TrackNodeConnection(int64 step, const GraphNode* node, - const GraphNode* src); - const std::map& devices() const { return devices_; } private: @@ -130,13 +134,9 @@ class Timeline { void GenerateCodeTimeline(const CodeNode* node); + private: void TrackNode(const GraphNode* node) { mem_tracker_.TrackNode(step_, node); } - void TrackNodeConnection(GraphNode* node, GraphNode* src) { - mem_tracker_.TrackNodeConnection(step_, node, src); - } - - private: void OutputTimeline(); template diff --git a/tensorflow/core/profiler/internal/tfprof_timeline_test.cc b/tensorflow/core/profiler/internal/tfprof_timeline_test.cc index babae395bad44ce72f95d20f14f455c3697f8c63..91eac0cf7617eba54f6938fb893192d2a8fe2eaf 100644 --- a/tensorflow/core/profiler/internal/tfprof_timeline_test.cc +++ b/tensorflow/core/profiler/internal/tfprof_timeline_test.cc @@ -71,7 +71,7 @@ TEST_F(TFProfTimelineTest, GraphView) { string dump_str; TF_CHECK_OK(ReadFileToString(Env::Default(), dump_file + "_0", &dump_str)); - EXPECT_EQ(1754536562981488144ull, Hash64(dump_str)); + EXPECT_EQ(7932146665024565912ull, Hash64(dump_str)); } TEST_F(TFProfTimelineTest, ScopeView) { diff --git a/tensorflow/core/profiler/profiler.cc b/tensorflow/core/profiler/profiler.cc index 96e0b06bf3dfb2de8cf462fff0f91b32bf837885..a5e513aa21c56e605681aaf7e5d46815a820cec7 100644 --- a/tensorflow/core/profiler/profiler.cc +++ b/tensorflow/core/profiler/profiler.cc @@ -234,6 +234,7 @@ int Run(int argc, char** argv) { return 1; } tf_stat->AddRunMeta(i, std::move(run_meta)); + fprintf(stdout, "run graph coverage: %.2f\n", tf_stat->run_coverage()); } } diff --git a/tensorflow/core/profiler/tfprof_log.proto b/tensorflow/core/profiler/tfprof_log.proto index a1410c7c79e22debf5c7672833b48d53e0e461ba..f92301133a3102a2e4233326dd811169e1ecd105 100644 --- a/tensorflow/core/profiler/tfprof_log.proto +++ b/tensorflow/core/profiler/tfprof_log.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package tensorflow.tfprof; import "tensorflow/core/framework/attr_value.proto"; +import "tensorflow/core/framework/step_stats.proto"; // It specifies the Python callstack that creates an op. message CodeDef { @@ -89,6 +90,10 @@ message ProfileNode { map execs = 12; } +message Allocation { + repeated AllocationRecord allocation_records = 1; +} + message ExecProfile { // Can be larger than 1 if run multiple times in loop. int64 run_count = 1; @@ -107,6 +112,8 @@ message ExecProfile { map output_memory = 17; + repeated Allocation allocations = 18; + repeated string devices = 6; // Total bytes requested by the op. diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto index 56bb709e11943c9c16ed0e317afdeb250333118c..145311b59d9c9455bfe78fe83a005231e306c62e 100644 --- a/tensorflow/core/protobuf/config.proto +++ b/tensorflow/core/protobuf/config.proto @@ -87,6 +87,13 @@ message OptimizerOptions { // If true, perform constant folding optimization on the graph. bool do_constant_folding = 2; + // Constant folding optimization replaces tensors whose values can be + // predetermined, with constant nodes. To avoid inserting too large constants, + // the size of each constant created can be limited. If this value is zero, a + // default limit of 10 MiB will be applied. If constant folding optimization + // is disabled, this value is ignored. + int64 max_folded_constant_in_bytes = 6; + // If true, perform function inlining on the graph. bool do_function_inlining = 4; diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 5d2298f7b7dbe6e97eb5b8577fc8920837044b9d..95ada559fddc5d6e87ca5778e7dfc2a5119c41c0 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -24,7 +24,7 @@ limitations under the License. // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") -#define TF_VERSION_SUFFIX "-rc0" +#define TF_VERSION_SUFFIX "-rc1" #define TF_STR_HELPER(x) #x #define TF_STR(x) TF_STR_HELPER(x) @@ -116,5 +116,7 @@ extern const char* tf_compiler_version(); // The git commit designator when tensorflow was built // If no git repository, this will be "internal". extern const char* tf_git_version(); +// Value of the _GLIBCXX_USE_CXX11_ABI flag, or -1 if it's not set. +extern const int tf_cxx11_abi_flag(); #endif // TENSORFLOW_CORE_PUBLIC_VERSION_H_ diff --git a/tensorflow/core/util/tensor_bundle/BUILD b/tensorflow/core/util/tensor_bundle/BUILD index 4e957ec3df1f42dc0360baa8f731e46e6c73d551..166bd0f659dae3124faac6d71d69cbcd41c15b48 100644 --- a/tensorflow/core/util/tensor_bundle/BUILD +++ b/tensorflow/core/util/tensor_bundle/BUILD @@ -66,6 +66,7 @@ tf_cc_test( srcs = ["tensor_bundle_test.cc"], deps = [ ":tensor_bundle", + "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", "//tensorflow/core:protos_all_cc", diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc index 02eb042a0b73278818a6ee7926546a022c587194..d0e54b7e4774e8cd2b2295df4f3fa4c724acbfac 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc @@ -22,10 +22,14 @@ limitations under the License. #include #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/framework/tensor_shape.pb_text.h" #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb_text.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_op_registry.h" +#include "tensorflow/core/framework/variant_tensor_data.h" #include "tensorflow/core/framework/versions.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/lib/core/coding.h" @@ -109,6 +113,64 @@ Status ReadStringTensor(io::InputBuffer* buffered_file, size_t num_elements, return Status::OK(); } +Status ReadVariantTensor(io::InputBuffer* buffered_file, Tensor* ret, + size_t offset, size_t size, uint32* actual_crc32c) { + // On-disk format: + // [varint64 len1][bytes variant1][4 byte checksum] + // .. + // [varint64 lenN][bytes variantN][4 byte checksum] + // Var "crc32c" checksums all the lens, variant bytes, individual variant + // checksums (as uint32, not varint32 bytes). + if (size == 0) return Status::OK(); + size_t num_elements = ret->NumElements(); + + // Reads the actual string bytes. + TF_RETURN_IF_ERROR(buffered_file->Seek(offset)); + for (size_t i = 0; i < num_elements; ++i) { + // Read the serialized variant length. + uint64 string_length = 0; + TF_RETURN_IF_ERROR(buffered_file->ReadVarint64(&string_length)); + *actual_crc32c = crc32c::Extend( + *actual_crc32c, reinterpret_cast(&string_length), + sizeof(uint64)); + // Read the actual serialized variant. + string buffer; + buffer.resize(string_length); + size_t bytes_read = 0; + TF_RETURN_IF_ERROR( + buffered_file->ReadNBytes(string_length, &buffer[0], &bytes_read)); + *actual_crc32c = crc32c::Extend(*actual_crc32c, buffer.data(), bytes_read); + VariantTensorDataProto proto; + proto.ParseFromString(buffer); + Variant v = proto; + if (!DecodeUnaryVariant(&v)) { + return errors::Internal("Could not decode variant with type_name: \"", + v.TypeName(), "\". Perhaps you forgot to ", + "register a decoder via ", + "REGISTER_UNARY_VARIANT_DECODE_FUNCTION?"); + } + + // Read the checksum. + uint32 checksum = 0; + size_t unused_bytes_read = 0; + TF_RETURN_IF_ERROR(buffered_file->ReadNBytes( + sizeof(uint32), reinterpret_cast(&checksum), + &unused_bytes_read)); + if (crc32c::Unmask(checksum) != *actual_crc32c) { + return errors::DataLoss( + "The checksum after Variant ", i, " does not match.", + " Expected: ", strings::Printf("%08u", crc32c::Unmask(checksum)), + " Actual: ", strings::Printf("%08u", *actual_crc32c)); + } + *actual_crc32c = crc32c::Extend( + *actual_crc32c, reinterpret_cast(&checksum), sizeof(uint32)); + + ret->flat()(i) = std::move(v); + } + + return Status::OK(); +} + char* GetBackingBuffer(const Tensor& val) { CHECK(DataTypeCanUseMemcpy(val.dtype())) << val.dtype(); return const_cast(val.tensor_data().data()); @@ -134,6 +196,7 @@ Status ParseEntryProto(StringPiece key, StringPiece value, Status WriteTensor(const Tensor& val, FileOutputBuffer* out, size_t* bytes_written) { DCHECK_NE(val.dtype(), DT_STRING); + DCHECK_NE(val.dtype(), DT_VARIANT); *bytes_written = val.TotalBytes(); char* buf = GetBackingBuffer(val); VLOG(1) << "Appending " << *bytes_written << " bytes to file"; @@ -188,6 +251,54 @@ Status WriteStringTensor(const Tensor& val, FileOutputBuffer* out, return Status::OK(); } +Status WriteVariantTensor(const Tensor& val, FileOutputBuffer* out, + size_t* bytes_written, uint32* crc32c) { + // On-disk format: + // [varint64 len1][bytes variant1][4 byte checksum] + // .. + // [varint64 lenN][bytes variantN][4 byte checksum] + // Var "crc32c" checksums all the lens, variant bytes, individual variant + // checksums (as uint32, not varint32 bytes). + DCHECK_EQ(val.dtype(), DT_VARIANT); + + *crc32c = 0; + *bytes_written = 0; + for (int64 i = 0; i < val.NumElements(); ++i) { + VariantTensorData data; + val.flat()(i).Encode(&data); + VariantTensorDataProto proto; + data.ToProto(&proto); + string elem; + proto.SerializeToString(&elem); + + // Write the length of the serialized variant. + DCHECK_EQ(elem.size(), static_cast(elem.size())); + const auto elem_size = static_cast(elem.size()); + string len; + core::PutVarint64(&len, elem_size); + TF_RETURN_IF_ERROR(out->Append(len)); + *crc32c = crc32c::Extend(*crc32c, reinterpret_cast(&elem_size), + sizeof(uint64)); + *bytes_written += sizeof(uint64); + + // Write the serialized variant. + TF_RETURN_IF_ERROR(out->Append(elem)); + *crc32c = crc32c::Extend(*crc32c, elem.data(), elem.size()); + *bytes_written += elem.size(); + + // Write the checksum. + const uint32 length_checksum = crc32c::Mask(*crc32c); + TF_RETURN_IF_ERROR(out->Append(StringPiece( + reinterpret_cast(&length_checksum), sizeof(uint32)))); + *crc32c = + crc32c::Extend(*crc32c, reinterpret_cast(&length_checksum), + sizeof(uint32)); + *bytes_written += sizeof(uint32); + } + + return Status::OK(); +} + // Reads file[offset:offset+size) into destination[0:size). Each Read() copies // at most "buffer_size" bytes. // @@ -312,11 +423,13 @@ Status BundleWriter::Add(StringPiece key, const Tensor& val) { size_t data_bytes_written = 0; uint32 crc32c = 0; out_->clear_crc32c(); - if (val.dtype() != DT_STRING) { + if (val.dtype() == DT_STRING) { + status_ = WriteStringTensor(val, out_.get(), &data_bytes_written, &crc32c); + } else if (val.dtype() == DT_VARIANT) { + status_ = WriteVariantTensor(val, out_.get(), &data_bytes_written, &crc32c); + } else { status_ = WriteTensor(val, out_.get(), &data_bytes_written); crc32c = out_->crc32c(); - } else { - status_ = WriteStringTensor(val, out_.get(), &data_bytes_written, &crc32c); } if (status_.ok()) { @@ -707,13 +820,13 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { } // Validates the "size" field. - if (entry.dtype() != DT_STRING) { + if (entry.dtype() != DT_STRING && entry.dtype() != DT_VARIANT) { if (entry.size() != ret->TotalBytes()) { return errors::DataLoss("Invalid size in bundle entry: key ", key(), "; stored size ", entry.size(), "; expected size ", ret->TotalBytes()); } - } else { + } else if (entry.dtype() == DT_STRING) { // Relaxes the check for string tensors as follows: // entry.size() == bytes(varint lengths) + bytes(data) // >= NumElems + bytes(data), since size bytes(varint) >= 1. @@ -752,6 +865,11 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { entry.size(), 8 << 20 /* 8MB buffer */, backing_buffer)); actual_crc32c = crc32c::Value(backing_buffer, entry.size()); + } else if (entry.dtype() == DT_VARIANT) { + // Relies on io::InputBuffer's buffering, because we issue many neighboring + // reads for a single string tensor. + TF_RETURN_IF_ERROR(ReadVariantTensor(buffered_file, ret, entry.offset(), + entry.size(), &actual_crc32c)); } else { // Relies on io::InputBuffer's buffering, because we issue many neighboring // reads for a single string tensor. diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc index 4ee15785108a8a0618f97ff3db0aaee26de15368..341aae36f4165767d56f28bcf733146f473c897b 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/framework/versions.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" @@ -64,6 +66,30 @@ void Expect(BundleReader* reader, const string& key, test::ExpectTensorEqual(val, expected_val); } +template +void ExpectVariant(BundleReader* reader, const string& key, + const Tensor& expected_t) { + // Tests for Contains(). + EXPECT_TRUE(reader->Contains(key)); + // Tests for LookupDtypeAndShape(). + DataType dtype; + TensorShape shape; + TF_ASSERT_OK(reader->LookupDtypeAndShape(key, &dtype, &shape)); + // Tests for Lookup(), checking tensor contents. + EXPECT_EQ(expected_t.dtype(), dtype); + EXPECT_EQ(expected_t.shape(), shape); + Tensor actual_t(dtype, shape); + TF_ASSERT_OK(reader->Lookup(key, &actual_t)); + for (int i = 0; i < expected_t.NumElements(); i++) { + Variant actual_var = actual_t.flat()(i); + Variant expected_var = expected_t.flat()(i); + EXPECT_EQ(actual_var.TypeName(), expected_var.TypeName()); + auto* actual_val = actual_var.get(); + auto* expected_val = expected_var.get(); + EXPECT_EQ(*expected_val, *actual_val); + } +} + template void ExpectNext(BundleReader* reader, const Tensor& expected_val) { EXPECT_TRUE(reader->Valid()); @@ -460,6 +486,55 @@ TEST(TensorBundleTest, StringTensors) { } } +class VariantObject { + public: + VariantObject() {} + VariantObject(const string& metadata, int64 value) + : metadata_(metadata), value_(value) {} + + string TypeName() const { return "TEST VariantObject"; } + void Encode(VariantTensorData* data) const { + data->set_type_name(TypeName()); + data->set_metadata(metadata_); + Tensor val_t = Tensor(DT_INT64, TensorShape({})); + val_t.scalar()() = value_; + *(data->add_tensors()) = val_t; + } + bool Decode(const VariantTensorData& data) { + EXPECT_EQ(data.type_name(), TypeName()); + data.get_metadata(&metadata_); + EXPECT_EQ(data.tensors_size(), 1); + value_ = data.tensors(0).scalar()(); + return true; + } + bool operator==(const VariantObject other) const { + return metadata_ == other.metadata_ && value_ == other.value_; + } + string metadata_; + int64 value_; +}; + +REGISTER_UNARY_VARIANT_DECODE_FUNCTION(VariantObject, "TEST VariantObject"); + +TEST(TensorBundleTest, VariantTensors) { + { + BundleWriter writer(Env::Default(), Prefix("foo")); + TF_EXPECT_OK( + writer.Add("variant_tensor", + test::AsTensor({VariantObject("test", 10), + VariantObject("test1", 20)}))); + TF_ASSERT_OK(writer.Finish()); + } + { + BundleReader reader(Env::Default(), Prefix("foo")); + TF_ASSERT_OK(reader.status()); + ExpectVariant( + &reader, "variant_tensor", + test::AsTensor( + {VariantObject("test", 10), VariantObject("test1", 20)})); + } +} + TEST(TensorBundleTest, DirectoryStructure) { Env* env = Env::Default(); // Writes two bundles. diff --git a/tensorflow/docs_src/api_guides/cc/guide.md b/tensorflow/docs_src/api_guides/cc/guide.md index f30bf3797edf4c345eeb29e4268229154fce11b0..81fb1e1fda277e8035ada5a410b966fe2de35a09 100644 --- a/tensorflow/docs_src/api_guides/cc/guide.md +++ b/tensorflow/docs_src/api_guides/cc/guide.md @@ -1,4 +1,12 @@ # C++ API + +Note: By default [tensorflow.org](http://tensorflow.org) shows docs for the +most recent stable version. The instructions in this doc require building from +source. You will probably want to build from the `master` version of tensorflow. +You should, as a result, be sure you are following the +[`master` version of this doc](https://www.tensorflow.org/versions/master/api_guides/cc/guide), +in case there have been any changes. + [TOC] TensorFlow's C++ API provides mechanisms for constructing and executing a data @@ -48,7 +56,9 @@ TensorFlow `BUILD` file in the same directory with the following contents: ```python -cc_binary( +load("//tensorflow:tensorflow.bzl", "tf_cc_binary") + +tf_cc_binary( name = "example", srcs = ["example.cc"], deps = [ @@ -59,8 +69,10 @@ cc_binary( ) ``` -You should be able to build and run the example using the following command -(be sure to run `./configure` in your build sandbox first): +Use `tf_cc_binary` rather than Bazel's native `cc_binary` to link in necessary +symbols from `libtensorflow_framework.so`. You should be able to build and run +the example using the following command (be sure to run `./configure` in your +build sandbox first): ```shell bazel run -c opt //tensorflow/cc/example:example diff --git a/tensorflow/docs_src/api_guides/python/contrib.metrics.md b/tensorflow/docs_src/api_guides/python/contrib.metrics.md index b502826e6ac63d95c4912339e0ca9d2fa7f900a7..1eb9cf417a3c8e9926b6d588b14524efd10f12df 100644 --- a/tensorflow/docs_src/api_guides/python/contrib.metrics.md +++ b/tensorflow/docs_src/api_guides/python/contrib.metrics.md @@ -64,7 +64,7 @@ sess.run(tf.local_variables_initializer()) for batch in range(num_batches): sess.run([update_op_acc, update_op_error]) -accuracy, mean_absolute_error = sess.run([accuracy, mean_absolute_error]) +accuracy, error = sess.run([accuracy, error]) ``` Note that when evaluating the same metric multiple times on different inputs, diff --git a/tensorflow/docs_src/api_guides/python/input_dataset.md b/tensorflow/docs_src/api_guides/python/input_dataset.md new file mode 100644 index 0000000000000000000000000000000000000000..2798d76be988e5b340ebcb717910d63201e7caf8 --- /dev/null +++ b/tensorflow/docs_src/api_guides/python/input_dataset.md @@ -0,0 +1,81 @@ +# `Dataset` Input Pipeline +[TOC] + +@{tf.data.Dataset} allows you to build complex input pipelines. See the +@{$datasets$programmer's guide} for an in-depth explanation of how to use this +API. + +## Reader classes + +Classes that create a dataset from input files. + +* @{tf.data.FixedLengthRecordDataset} +* @{tf.data.TextLineDataset} +* @{tf.data.TFRecordDataset} + +## Creating new datasets + +Static methods in `Dataset` that create new datasets. + +* @{tf.data.Dataset.from_generator} +* @{tf.data.Dataset.from_sparse_tensor_slices} +* @{tf.data.Dataset.from_tensor_slices} +* @{tf.data.Dataset.from_tensors} +* @{tf.data.Dataset.list_files} +* @{tf.data.Dataset.range} +* @{tf.data.Dataset.zip} + +## Transformations on existing datasets + +These functions transform an existing dataset, and return a new dataset. Calls +can be chained together, as shown in the example below: + +``` +train_data = train_data.batch(100).shuffle().repeat() +``` + +* @{tf.data.Dataset.apply} +* @{tf.data.Dataset.batch} +* @{tf.data.Dataset.cache} +* @{tf.data.Dataset.concatenate} +* @{tf.data.Dataset.filter} +* @{tf.data.Dataset.flat_map} +* @{tf.data.Dataset.interleave} +* @{tf.data.Dataset.map} +* @{tf.data.Dataset.padded_batch} +* @{tf.data.Dataset.prefetch} +* @{tf.data.Dataset.repeat} +* @{tf.data.Dataset.shard} +* @{tf.data.Dataset.shuffle} +* @{tf.data.Dataset.skip} +* @{tf.data.Dataset.take} + +### Custom transformation functions + +Custom transformation functions can be applied to a `Dataset` using @{tf.data.Dataset.apply}. Below are custom transformation functions from `tf.contrib.data`: + +* @{tf.contrib.data.batch_and_drop_remainder} +* @{tf.contrib.data.dense_to_sparse_batch} +* @{tf.contrib.data.enumerate_dataset} +* @{tf.contrib.data.group_by_window} +* @{tf.contrib.data.ignore_errors} +* @{tf.contrib.data.rejection_resample} +* @{tf.contrib.data.sloppy_interleave} +* @{tf.contrib.data.unbatch} + +## Iterating over datasets + +These functions make a @{tf.data.Iterator} from a `Dataset`. + +* @{tf.data.Dataset.make_initializable_iterator} +* @{tf.data.Dataset.make_one_shot_iterator} + +The `Iterator` class also contains static methods that create a @{tf.data.Iterator} that can be used with multiple `Dataset` objects. + +* @{tf.data.Iterator.from_structure} +* @{tf.data.Iterator.from_string_handle} + +## Extra functions from `tf.contrib.data` + +* @{tf.contrib.data.read_batch_features} + diff --git a/tensorflow/docs_src/api_guides/python/reading_data.md b/tensorflow/docs_src/api_guides/python/reading_data.md index e7fb05f9b5646910cc6d0e55218c55bfba219d94..b3ebaa0f0a3645256d4e92632a10a53e4eb243cb 100644 --- a/tensorflow/docs_src/api_guides/python/reading_data.md +++ b/tensorflow/docs_src/api_guides/python/reading_data.md @@ -3,16 +3,25 @@ Note: The preferred way to feed data into a tensorflow program is using the @{$datasets$Datasets API}. -There are three other methods of getting data into a TensorFlow program: +There are four methods of getting data into a TensorFlow program: +* `Dataset` API: Easily construct a complex input pipeline. (preferred method) * Feeding: Python code provides the data when running each step. -* Reading from files: an input pipeline reads the data from files +* `QueueRunner`: a queue-based input pipeline reads the data from files at the beginning of a TensorFlow graph. * Preloaded data: a constant or variable in the TensorFlow graph holds all the data (for small data sets). [TOC] +## Dataset API + +See the @{$datasets$programmer's guide} for an in-depth explanation of +@{tf.data.Dataset}. The `Dataset` API allows you to extract and preprocess data +from different input/file formats, and apply transformations such as batch, +shuffle, and map to the dataset. This is an improved version of the old input +methods, feeding and `QueueRunner`. + ## Feeding TensorFlow's feed mechanism lets you inject data into any Tensor in a @@ -22,7 +31,7 @@ graph. Supply feed data through the `feed_dict` argument to a run() or eval() call that initiates computation. -Note: "Feeding" is the least efficient way to feed data into a tensorflow +Warning: "Feeding" is the least efficient way to feed data into a tensorflow program and should only be used for small experiments and debugging. ```python @@ -44,9 +53,9 @@ in [`tensorflow/examples/tutorials/mnist/fully_connected_feed.py`](https://www.tensorflow.org/code/tensorflow/examples/tutorials/mnist/fully_connected_feed.py), and is described in the @{$mechanics$MNIST tutorial}. -## Reading from files +## `QueueRunner` -A typical pipeline for reading records from files has the following stages: +A typical queue-based pipeline for reading records from files has the following stages: 1. The list of filenames 2. *Optional* filename shuffling @@ -57,7 +66,7 @@ A typical pipeline for reading records from files has the following stages: 7. *Optional* preprocessing 8. Example queue -Note: This section discusses implementing input pipelines using the +Warning: This section discusses implementing input pipelines using the queue-based APIs which can be cleanly replaced by the @{$datasets$Datasets API}. ### Filenames, shuffling, and epoch limits diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md index 655506b09824a9f0911155269a869058a572f4e2..77d4e0caece4b50222c6e8abdd7ebba006159f26 100644 --- a/tensorflow/docs_src/community/documentation.md +++ b/tensorflow/docs_src/community/documentation.md @@ -1,6 +1,6 @@ # Writing TensorFlow Documentation -We welcome contributions to the Tensorflow documentation from the community. +We welcome contributions to the TensorFlow documentation from the community. This document explains how you can contribute to that documentation. In particular, this document explains the following: @@ -8,28 +8,30 @@ particular, this document explains the following: * How to make conformant edits. * How to build and test your documentation changes before you submit them. -You can view Tensorflow documentation on https://www.tensorflow.org, and you -can view and edit the raw files on Github. We're publishing our docs on Github -so everybody can contribute. Whatever gets checked in tensorflow/docs_src will -be published soon after on https://www.tensorflow.org. +You can view TensorFlow documentation on https://www.tensorflow.org, and you +can view and edit the raw files on +[GitHub](https://www.tensorflow.org/code/tensorflow/docs_src/). +We're publishing our docs on GitHub so everybody can contribute. Whatever gets +checked in to `tensorflow/docs_src` will be published soon after on +https://www.tensorflow.org. Republishing TensorFlow documentation in different forms is absolutely allowed, but we are unlikely to accept other documentation formats (or the tooling to generate them) into our repository. If you do choose to republish our documentation in another form, please be sure to include: -* The version of the API this represents (i.e. r1.0, master, etc.) +* The version of the API this represents (for example, r1.0, master, etc.) * The commit or version from which the documentation was generated * Where to get the latest documentation (that is, https://www.tensorflow.org) * The Apache 2.0 license. -## A Note on Versions +## A note on versions tensorflow.org, at root, shows documentation for the latest stable binary. This is the documentation you should be reading if you are using `pip` to install TensorFlow. -However, most developers will contribute documentation into the master Github +However, most developers will contribute documentation into the master GitHub branch, which is published, occasionally, at [tensorflow.org/versions/master](https://www.tensorflow.org/versions/master). @@ -49,8 +51,9 @@ in the code: To modify the reference documentation, you edit the appropriate code comments. Non-reference documentation (for example, the TensorFlow installation guides) is -authored by humans. This documentation is located in the `tensorflow/docs_src` -directory. Each subdirectory of `docs_src` contains a set of related Tensorflow +authored by humans. This documentation is located in the +[`tensorflow/docs_src`](https://www.tensorflow.org/code/tensorflow/docs_src/) +directory. Each subdirectory of `docs_src` contains a set of related TensorFlow documentation. For example, the TensorFlow installation guides are all in the `docs_src/install` directory. @@ -183,7 +186,7 @@ documentation in the `/tmp/tfdocs` dir: Note: You must set `src_dir` and `output_dir` to absolute file paths. -## Generating Python API Documentation +## Generating Python API documentation Ops, classes, and utility functions are defined in Python modules, such as `image_ops.py`. Python modules contain a module docstring. For example: @@ -216,7 +219,7 @@ the following: Only top level modules (currently just `tf` and `tfdbg`) need to be manually added to the generate script. -### Sealing Modules +### Sealing modules Because the doc generator walks all visible symbols, and descends into anything it finds, it will document any accidentally exposed symbols. If a module only @@ -242,7 +245,7 @@ following options for dealing with them: We'll discuss these options in detail below. -#### Private Symbols and Imports +#### Private symbols and imports The easiest way to conform to the API sealing expectations is to make non-public symbols private (by prepending an underscore _). The doc generator respects @@ -288,7 +291,7 @@ are public. All `@@`s will eventually be removed. If you see them, however, please do not randomly delete them as they are still in use by some of our systems. -#### Traversal Blacklist +#### Traversal blacklist If all else fails, you may add entries to the traversal blacklist in `generate_lib.py.` **Almost all entries in this list are an abuse of its @@ -311,7 +314,7 @@ flags, ...) included for platform abstraction can be documented without documenting their interior. Its use beyond this purpose is a shortcut that may be acceptable for contrib, but not for core tensorflow. -## Op Documentation Style Guide +## Op documentation style guide Long, descriptive module-level documentation for modules should go in the API Guides in `docs_src/api_guides/python`. @@ -334,7 +337,7 @@ is [here](https://daringfireball.net/projects/markdown/). You are allowed to use [MathJax](https://www.mathjax.org) notation for equations (see above for restrictions). -### Writing About Code +### Writing about code Put backticks around these things when they're used in text: @@ -375,7 +378,7 @@ Two notes about backticks for code samples in Markdown: However, do NOT indent four spaces and use backticks simultaneously. Use one or the other. -### Tensor Dimensions +### Tensor dimensions When you're talking about a tensor in general, don't capitalize the word tensor. When you're talking about the specific object that's provided to an op as an @@ -500,7 +503,7 @@ def foo(x, y, name="bar"): """ ``` -## Description of the Docstring Sections +## Description of the docstring sections This section details each of the elements in docstrings. diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md index 4991783a53a5a5fd5168aca14e4cf7db6847e665..33740de5d5af11cb6a8f1f6d57baa4c0e0dbefff 100644 --- a/tensorflow/docs_src/community/welcome.md +++ b/tensorflow/docs_src/community/welcome.md @@ -20,7 +20,6 @@ The TensorFlow community has created many great projects around TensorFlow, incl * [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com) * [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow) * [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials) -* [Scikit Flow - Simplified Interface for TensorFlow](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/learn/python/learn) * [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow) * [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0)) * [Rust language bindings](https://github.com/google/tensorflow-rust) @@ -29,6 +28,7 @@ The TensorFlow community has created many great projects around TensorFlow, incl * [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow) * [Edward - A library for probabilistic modeling, inference, and criticism](http://edwardlib.org) ([Github](https://github.com/blei-lab/edward), [Forum](https://discourse.edwardlib.org)) * [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow) +* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - Please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource. ## TensorFlow Communities Around the World diff --git a/tensorflow/docs_src/extend/estimators.md b/tensorflow/docs_src/extend/estimators.md index 5defade7ae254674a9d138b81bb6bac6bc0cfd22..7e6507c5840fe621aeb91842c9a83554e568db99 100644 --- a/tensorflow/docs_src/extend/estimators.md +++ b/tensorflow/docs_src/extend/estimators.md @@ -44,7 +44,7 @@ feature columns, input functions, and `train()`/`evaluate()`/`predict()` operations. If you've never used tf.estimator before, or need a refresher, you should first review the following tutorials: -* @{$estimator$tf.estimator Quickstart}: Quick introduction to +* @{$get_started/estimator$tf.estimator Quickstart}: Quick introduction to training a neural network using tf.estimator. * @{$wide$TensorFlow Linear Model Tutorial}: Introduction to feature columns, and an overview on building a linear classifier in diff --git a/tensorflow/docs_src/extend/index.md b/tensorflow/docs_src/extend/index.md index 5812caaffc9c12d3719c0830002bb932d7f5a996..3f30b9a8c243728f6dd2a47ffa0b35fb92ee68fe 100644 --- a/tensorflow/docs_src/extend/index.md +++ b/tensorflow/docs_src/extend/index.md @@ -14,7 +14,7 @@ TensorFlow: add support for your own shared or distributed filesystem. * @{$new_data_formats$Custom Data Readers}, which details how to add support for your own file and record formats. - * @{$estimators$Creating Estimators in tf.contrib.learn}, which explains how + * @{$extend/estimators$Creating Estimators in tf.contrib.learn}, which explains how to write your own custom Estimator. For example, you could build your own Estimator to implement some variation on standard linear regression. diff --git a/tensorflow/docs_src/get_started/estimator.md b/tensorflow/docs_src/get_started/estimator.md index 11c3dc6e53ea438cf112eb2f14d50b6983ad2657..790de6679b0bdbe5f91fd03e3ebfedc278b5b3c8 100644 --- a/tensorflow/docs_src/get_started/estimator.md +++ b/tensorflow/docs_src/get_started/estimator.md @@ -400,7 +400,7 @@ second sample is *Iris virginica*. @{$linear$Large-scale Linear Models with TensorFlow}. * To build your own Estimator using tf.estimator APIs, check out - @{$estimators$Creating Estimators in tf.estimator}. + @{$extend/estimators$Creating Estimators}. * To experiment with neural network modeling and visualization in the browser, check out [Deep Playground](http://playground.tensorflow.org/). diff --git a/tensorflow/docs_src/get_started/get_started.md b/tensorflow/docs_src/get_started/get_started.md index 67fddfe8094811ca1eed5bd1d80ccd8d56dda9f8..8409962744c71eb226af8d859922729b35bf6ad3 100644 --- a/tensorflow/docs_src/get_started/get_started.md +++ b/tensorflow/docs_src/get_started/get_started.md @@ -453,7 +453,7 @@ input_fn = tf.estimator.inputs.numpy_input_fn( train_input_fn = tf.estimator.inputs.numpy_input_fn( {"x": x_train}, y_train, batch_size=4, num_epochs=1000, shuffle=False) eval_input_fn = tf.estimator.inputs.numpy_input_fn( - {"x": x_eval}, y_eval, batch_size=4, num_epochs=1000, shuffle=False) + {"x": x_eval}, y_eval, batch_size=4, num_epochs=1, shuffle=False) # train estimator.train(input_fn=input_fn, steps=1000) diff --git a/tensorflow/docs_src/get_started/input_fn.md b/tensorflow/docs_src/get_started/input_fn.md index 7706c07b1d940f98acf89ecf63df5e9f7af31366..9d3af5d96a94d3f55dc82e64459b558630e6e7f0 100644 --- a/tensorflow/docs_src/get_started/input_fn.md +++ b/tensorflow/docs_src/get_started/input_fn.md @@ -11,7 +11,7 @@ median house values. The `input_fn` is used to pass feature and target data to the `train`, `evaluate`, and `predict` methods of the `Estimator`. The user can do feature engineering or pre-processing inside the `input_fn`. -Here's an example taken from the @{$estimator$tf.estimator Quickstart tutorial}: +Here's an example taken from the @{$get_started/estimator$tf.estimator Quickstart tutorial}: ```python import numpy as np diff --git a/tensorflow/docs_src/get_started/linear_regression.md b/tensorflow/docs_src/get_started/linear_regression.md index 7cfff8db15e0bbb4209794420db3ece816a95e5f..45cb9d829cfbc1b1efb735cc1ea27e33159db724 100644 --- a/tensorflow/docs_src/get_started/linear_regression.md +++ b/tensorflow/docs_src/get_started/linear_regression.md @@ -4,32 +4,28 @@ This unit provides the following short examples demonstrating how to implement regression in Estimators: - + - - - - @@ -96,7 +92,7 @@ During training, all three programs output the following information: For example, here's some possible output for the `linear_regressor.py` program: -```bsh +``` None INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpAObiz9/model.ckpt. INFO:tensorflow:loss = 161.308, step = 1 INFO:tensorflow:global_step/sec: 1557.24 diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md index 586bb6dead5911a62008669fe21c6872448df8be..3a153e81145676aab7e9f95f9d1c78fa7531a2cc 100644 --- a/tensorflow/docs_src/install/install_c.md +++ b/tensorflow/docs_src/install/install_c.md @@ -9,10 +9,13 @@ The API leans towards simplicity and uniformity rather than convenience. ## Supported Platforms -You may install TensorFlow for C on the following operating systems: +This guide explains how to install TensorFlow for C. Although these +instructions might also work on other variants, we have only tested +(and we only support) these instructions on machines meeting the +following requirements: - * Linux - * Mac OS X + * Linux, 64-bit, x86 + * macOS X, Version 10.11 (El Capitan) or higher ## Installation @@ -26,16 +29,16 @@ enable TensorFlow for C: following guides: * @{$install_linux#determine_which_tensorflow_to_install$Installing TensorFlow on Linux} - * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on Mac OS} + * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on macOS} 2. Download and extract the TensorFlow C library into `/usr/local/lib` by invoking the following shell commands: TF_TYPE="cpu" # Change to "gpu" for GPU support - OS="linux" # Change to "darwin" for Mac OS + OS="linux" # Change to "darwin" for macOS TARGET_DIRECTORY="/usr/local" curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.4.0-rc1.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` @@ -57,9 +60,9 @@ enable TensorFlow for C: directory (for example, `~/mydir/lib`) to two environment variables. For example: - 
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and Mac OS X
+     
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and macOS X
      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/mydir/lib # For Linux only
-     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For Mac OS X only

+     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For macOS X only
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md index 1d00661d837d8718f155d4f2cf2636b46da5f3b9..df43255896eb5084431be8336a5778d17607fd3f 100644 --- a/tensorflow/docs_src/install/install_go.md +++ b/tensorflow/docs_src/install/install_go.md @@ -5,16 +5,19 @@ well-suited to loading models created in Python and executing them within a Go application. This guide explains how to install and set up the [TensorFlow Go package](https://godoc.org/github.com/tensorflow/tensorflow/tensorflow/go). -**WARNING:** The TensorFlow Go API is *not* covered by the TensorFlow +Warning: The TensorFlow Go API is *not* covered by the TensorFlow [API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). ## Supported Platforms -You may install TensorFlow for Go on the following operating systems: +This guide explains how to install TensorFlow for Go. Although these +instructions might also work on other variants, we have only tested +(and we only support) these instructions on machines meeting the +following requirements: - * Linux - * Mac OS X + * Linux, 64-bit, x86 + * macOS X, 10.11 (El Capitan) or higher ## Installation @@ -27,7 +30,7 @@ steps to install this library and enable TensorFlow for Go: "Determine which TensorFlow to install" in one of the following guides: * @{$install_linux#determine_which_tensorflow_to_install$Installing TensorFlow on Linux} - * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on Mac OS} + * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on macOS} 2. Download and extract the TensorFlow C library into `/usr/local/lib` by invoking the following shell commands: @@ -35,7 +38,7 @@ steps to install this library and enable TensorFlow for Go: TF_TYPE="cpu" # Change to "gpu" for GPU support TARGET_DIRECTORY='/usr/local' curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.4.0-rc1.tar.gz" | sudo tar -C $TARGET_DIRECTORY -xz The `tar` command extracts the TensorFlow C library into the `lib` @@ -57,9 +60,9 @@ steps to install this library and enable TensorFlow for Go: directory (for example, `~/mydir/lib`) to two environment variables as follows: - 
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and Mac OS X
+     
 export LIBRARY_PATH=$LIBRARY_PATH:~/mydir/lib # For both Linux and macOS X
      export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/mydir/lib # For Linux only
-     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For Mac OS X only

+     export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:~/mydir/lib # For macOS X only
4. Now that the TensorFlow C library is installed, invoke `go get` as follows to download the appropriate packages and their dependencies: diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md index 3b3acfdcb3d1d1fd4ac5fc9459ed192f4cb22fd1..f7f2c3cdc71787a9ce93e323a29b07e6e6a7779d 100644 --- a/tensorflow/docs_src/install/install_java.md +++ b/tensorflow/docs_src/install/install_java.md @@ -6,18 +6,20 @@ Java application. This guide explains how to install [TensorFlow for Java](https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/package-summary) and use it in a Java application. -**WARNING:** The TensorFlow Java API is *not* covered by the TensorFlow +Warning: The TensorFlow Java API is *not* covered by the TensorFlow [API stability guarantees](https://www.tensorflow.org/programmers_guide/version_semantics). ## Supported Platforms -TensorFlow for Java is supported on the following operating systems: +This guide explains how to install TensorFlow for Java. Although these +instructions might also work on other variants, we have only tested +(and we only support) these instructions on machines meeting the +following requirements: - * Linux - * Mac OS X - * Windows - * Android + * Ubuntu 14.04 or higher; 64-bit, x86 + * macOS X 10.11 (El Capitan) or higher + * Windows 7 or higher; 64-bit, x86 The installation instructions for Android are in a separate [Android TensorFlow Support page](https://www.tensorflow.org/code/tensorflow/contrib/android). @@ -34,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs: org.tensorflow tensorflow - 1.4.0-rc0 + 1.4.0-rc1 ``` @@ -63,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow: org.tensorflow tensorflow - 1.4.0-rc0 + 1.4.0-rc1 @@ -81,14 +83,14 @@ As an example, these steps will create a Maven project that uses TensorFlow: public static void main(String[] args) throws Exception { try (Graph g = new Graph()) { final String value = "Hello from " + TensorFlow.version(); - + // Construct the computation graph with a single operation, a constant // named "MyConst" with a value "value". try (Tensor t = Tensor.create(value.getBytes("UTF-8"))) { // The Java API doesn't yet include convenience functions for adding operations. g.opBuilder("Const", "MyConst").setAttr("dtype", t.dataType()).setAttr("value", t).build(); } - + // Execute the "MyConst" operation in a Session. try (Session s = new Session(g); Tensor output = s.runner().fetch("MyConst").run().get(0)) { @@ -117,12 +119,12 @@ This section describes how to use TensorFlow using the `java` and `javac` commands from a JDK installation. If your project uses Apache Maven, then refer to the simpler instructions above instead. -### Install on Linux or Mac OS +### Install on Linux or macOS -Take the following steps to install TensorFlow for Java on Linux or Mac OS: +Take the following steps to install TensorFlow for Java on Linux or macOS: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc1.jar), which is the TensorFlow Java Archive (JAR). 2. Decide whether you will run TensorFlow for Java on CPU(s) only or with @@ -130,7 +132,7 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS: "Determine which TensorFlow to install" in one of the following guides: * @{$install_linux#determine_which_tensorflow_to_install$Installing TensorFlow on Linux} - * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on Mac OS} + * @{$install_mac#determine_which_tensorflow_to_install$Installing TensorFlow on macOS} 3. Download and extract the appropriate Java Native Interface (JNI) file for your operating system and processor support by running the @@ -141,7 +143,7 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS: OS=$(uname -s | tr '[:upper:]' '[:lower:]') mkdir -p ./jni curl -L \ - "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc0.tar.gz" | + "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.4.0-rc1.tar.gz" | tar -xz -C ./jni ### Install on Windows @@ -149,10 +151,10 @@ Take the following steps to install TensorFlow for Java on Linux or Mac OS: Take the following steps to install TensorFlow for Java on Windows: 1. Download - [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc0.jar), + [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.4.0-rc1.jar), which is the TensorFlow Java Archive (JAR). 2. Download the following Java Native Interface (JNI) file appropriate for - [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc0.zip). + [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.4.0-rc1.zip). 3. Extract this .zip file. @@ -200,7 +202,7 @@ must be part of your `classpath`. For example, you can include the downloaded `.jar` in your `classpath` by using the `-cp` compilation flag as follows: -
javac -cp libtensorflow-1.4.0-rc0.jar HelloTF.java
+
javac -cp libtensorflow-1.4.0-rc1.jar HelloTF.java
### Running @@ -212,13 +214,13 @@ two files are available to the JVM: * the extracted JNI library For example, the following command line executes the `HelloTF` program on Linux -and Mac OS X: +and macOS X: -
java -cp libtensorflow-1.4.0-rc0.jar:. -Djava.library.path=./jni HelloTF
+
java -cp libtensorflow-1.4.0-rc1.jar:. -Djava.library.path=./jni HelloTF
And the following command line executes the `HelloTF` program on Windows: -
java -cp libtensorflow-1.4.0-rc0.jar;. -Djava.library.path=jni HelloTF
+
java -cp libtensorflow-1.4.0-rc1.jar;. -Djava.library.path=jni HelloTF
If the program prints Hello from version, you've successfully installed TensorFlow for Java and are ready to use the API. If the program diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md index 2b488cc4f537da6fdb22fcb8ab268a6384637364..414ab7b1f7def3d43b717c628979b291fd9244f0 100644 --- a/tensorflow/docs_src/install/install_linux.md +++ b/tensorflow/docs_src/install/install_linux.md @@ -1,8 +1,12 @@ # Installing TensorFlow on Ubuntu -This guide explains how to install TensorFlow on Ubuntu. These instructions -might also work on other Linux variants, but we have only tested (and we -only support) these instructions on Ubuntu 14.04 or higher. +This guide explains how to install TensorFlow on Ubuntu. Although these +instructions might also work on other Linux variants, we have only +tested (and we only support) these instructions on machines meeting the +following requirements: + + * 64-bit desktops or laptops + * Ubuntu 14.04 or higher ## Determine which TensorFlow to install @@ -77,22 +81,22 @@ TensorFlow with GPU support, but only if you do the following: You must pick the mechanism by which you install TensorFlow. The supported choices are as follows: - * [virtualenv](#InstallingVirtualenv) + * [Virtualenv](#InstallingVirtualenv) * ["native" pip](#InstallingNativePip) * [Docker](#InstallingDocker) * [Anaconda](#InstallingAnaconda) * installing from sources, which is documented in [a separate guide](https://www.tensorflow.org/install/install_sources). -**We recommend the virtualenv installation.** +**We recommend the Virtualenv installation.** [Virtualenv](https://virtualenv.pypa.io/en/stable/) is a virtual Python environment isolated from other Python development, incapable of interfering with or being affected by other Python programs -on the same machine. During the virtualenv installation process, +on the same machine. During the Virtualenv installation process, you will install not only TensorFlow but also all the packages that TensorFlow requires. (This is actually pretty easy.) To start working with TensorFlow, you simply need to "activate" the -virtual environment. All in all, virtualenv provides a safe and +virtual environment. All in all, Virtualenv provides a safe and reliable mechanism for installing and running TensorFlow. Native pip installs TensorFlow directly on your system without going @@ -121,30 +125,30 @@ Use that package at your own risk. -## Installing with virtualenv +## Installing with Virtualenv Take the following steps to install TensorFlow with Virtualenv: - 1. Install pip and virtualenv by issuing one of the following commands: + 1. Install pip and Virtualenv by issuing one of the following commands: 
$ sudo apt-get install python-pip python-dev python-virtualenv # for Python 2.7
-     $ sudo apt-get install python3-pip python3-dev python-virtualenv # for Python 3.n
+ $ sudo apt-get install python3-pip python3-dev python-virtualenv # for Python 3.n - 2. Create a virtualenv environment by issuing one of the following commands: + 2. Create a Virtualenv environment by issuing one of the following commands: 
$ virtualenv --system-site-packages targetDirectory # for Python 2.7
-     $ virtualenv --system-site-packages -p python3 targetDirectory # for Python 3.n
+ $ virtualenv --system-site-packages -p python3targetDirectory # for Python 3.n where targetDirectory specifies the top of the - virtualenv tree. Our instructions assume that + Virtualenv tree. Our instructions assume that targetDirectory is `~/tensorflow`, but you may choose any directory. - 3. Activate the virtualenv environment by issuing one of the following + 3. Activate the Virtualenv environment by issuing one of the following commands: 
$ source ~/tensorflow/bin/activate # bash, sh, ksh, or zsh
-     $ source ~/tensorflow/bin/activate.csh  # csh or tcsh
+ $ source ~/tensorflow/bin/activate.csh # csh or tcsh The preceding source command should change your prompt to the following: @@ -156,22 +160,22 @@ Take the following steps to install TensorFlow with Virtualenv: 
(tensorflow)$ easy_install -U pip
5. Issue one of the following commands to install TensorFlow in the active - virtualenv environment: + Virtualenv environment: 
(tensorflow)$ pip install --upgrade tensorflow      # for Python 2.7
-     (tensorflow)$ pip3 install --upgrade tensorflow     # for Python 3.n
-     (tensorflow)$ pip install --upgrade tensorflow-gpu  # for Python 2.7 and GPU
-     (tensorflow)$ pip3 install --upgrade tensorflow-gpu # for Python 3.n and GPU
+ (tensorflow)$ pip3 install --upgrade tensorflow # for Python 3.n + (tensorflow)$ pip install --upgrade tensorflow-gpu # for Python 2.7 and GPU + (tensorflow)$ pip3 install --upgrade tensorflow-gpu # for Python 3.n and GPU - If the preceding command succeeds, skip Step 6. If the preceding + If the above command succeeds, skip Step 6. If the preceding command fails, perform Step 6. 6. (Optional) If Step 5 failed (typically because you invoked a pip version - lower than 8.1), install TensorFlow in the active virtualenv environment + lower than 8.1), install TensorFlow in the active Virtualenv environment by issuing a command of the following format: 
(tensorflow)$ pip install --upgrade tfBinaryURL   # Python 2.7
-     (tensorflow)$ pip3 install --upgrade tfBinaryURL  # Python 3.n
+ (tensorflow)$ pip3 install --upgradetfBinaryURL # Python 3.n where tfBinaryURL identifies the URL of the TensorFlow Python package. The appropriate value of @@ -181,10 +185,10 @@ Take the following steps to install TensorFlow with Virtualenv: [here](#the_url_of_the_tensorflow_python_package). For example, if you are installing TensorFlow for Linux, Python 3.4, and CPU-only support, issue the following command to install TensorFlow in the active - virtualenv environment: + Virtualenv environment: 
(tensorflow)$ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl If you encounter installation problems, see [Common Installation Problems](#common_installation_problems). @@ -195,14 +199,14 @@ If you encounter installation problems, see After installing TensorFlow, [validate the installation](#ValidateYourInstallation). -Note that you must activate the virtualenv environment each time you -use TensorFlow. If the virtualenv environment is not currently active, +Note that you must activate the Virtualenv environment each time you +use TensorFlow. If the Virtualenv environment is not currently active, invoke one of the following commands: -
$ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
+
 $ source ~/tensorflow/bin/activate      # bash, sh, ksh, or zsh
 $ source ~/tensorflow/bin/activate.csh  # csh or tcsh

 
-When the virtualenv environment is active, you may run
+When the Virtualenv environment is active, you may run
 TensorFlow programs from this shell.  Your prompt will become
 the following to indicate that your tensorflow environment is active:
 
@@ -265,9 +269,9 @@ take the following steps:
   1. Install TensorFlow by invoking **one** of the following commands:
 
      
$ pip install tensorflow      # Python 2.7; CPU support (no GPU support)
-     $ pip3 install tensorflow     # Python 3.n; CPU support (no GPU support)
-     $ pip install tensorflow-gpu  # Python 2.7;  GPU support
-     $ pip3 install tensorflow-gpu # Python 3.n; GPU support 

+    $ pip3 install tensorflow     # Python 3.n; CPU support (no GPU support)
+    $ pip install tensorflow-gpu  # Python 2.7;  GPU support
+    $ pip3 install tensorflow-gpu # Python 3.n; GPU support
If the preceding command runs to completion, you should now [validate your installation](#ValidateYourInstallation). @@ -276,7 +280,7 @@ take the following steps: by issuing a command of the following format: 
$ sudo pip  install --upgrade tfBinaryURL   # Python 2.7
-     $ sudo pip3 install --upgrade tfBinaryURL   # Python 3.n
+ $ sudo pip3 install --upgradetfBinaryURL # Python 3.n where tfBinaryURL identifies the URL of the TensorFlow Python package. The appropriate value of @@ -289,7 +293,7 @@ take the following steps: 
      $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl
If this step fails, see @@ -476,7 +480,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: 
      (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+ https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl @@ -490,11 +494,11 @@ To validate your TensorFlow installation, do the following: ### Prepare your environment -If you installed on native pip, virtualenv, or Anaconda, then +If you installed on native pip, Virtualenv, or Anaconda, then do the following: 1. Start a terminal. - 2. If you installed with virtualenv or Anaconda, activate your container. + 2. If you installed with Virtualenv or Anaconda, activate your container. 3. If you installed TensorFlow source code, navigate to any directory *except* one containing TensorFlow source code. @@ -644,14 +648,14 @@ This section documents the relevant values for Linux installations. CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp27-none-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp27-none-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -663,14 +667,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp34-cp34m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp34-cp34m-linux_x86_64.whl
Note that GPU support requires the NVIDIA hardware and software described in @@ -682,14 +686,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp35-cp35m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp35-cp35m-linux_x86_64.whl
@@ -701,14 +705,14 @@ Note that GPU support requires the NVIDIA hardware and software described in CPU only: 
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.4.0rc1-cp36-cp36m-linux_x86_64.whl
GPU support: 
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0dev-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.4.0rc1-cp36-cp36m-linux_x86_64.whl
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md index efd977089b5678f717dbd581815b86802bcb2b32..9a95710bfa78ab546bc9e7ff1c8bd33ccd8b23c8 100644 --- a/tensorflow/docs_src/install/install_mac.md +++ b/tensorflow/docs_src/install/install_mac.md @@ -1,6 +1,11 @@ # Installing TensorFlow on macOS -This guide explains how to install TensorFlow on macOS. +This guide explains how to install TensorFlow on macOS. Although these +instructions might also work on other macOS variants, we have only +tested (and we only support) these instructions on machines meeting the +following requirements: + + * macOS X 10.11 (El Capitan) or higher Note: As of version 1.2, TensorFlow no longer provides GPU support on macOS. @@ -8,21 +13,21 @@ Note: As of version 1.2, TensorFlow no longer provides GPU support on macOS. You must pick the mechanism by which you install TensorFlow. The supported choices are as follows: - * virtualenv + * Virtualenv * "native" pip * Docker * installing from sources, which is documented in [a separate guide](https://www.tensorflow.org/install/install_sources). -**We recommend the virtualenv installation.** +**We recommend the Virtualenv installation.** [Virtualenv](https://virtualenv.pypa.io/en/stable) is a virtual Python environment isolated from other Python development, incapable of interfering with or being affected by other Python programs -on the same machine. During the virtualenv installation process, +on the same machine. During the Virtualenv installation process, you will install not only TensorFlow but also all the packages that TensorFlow requires. (This is actually pretty easy.) To start working with TensorFlow, you simply need to "activate" the -virtual environment. All in all, virtualenv provides a safe and +virtual environment. All in all, Virtualenv provides a safe and reliable mechanism for installing and running TensorFlow. Native pip installs TensorFlow directly on your system without going through @@ -48,30 +53,30 @@ However, within Anaconda, we recommend installing TensorFlow with the That is, the TensorFlow team neither tests nor maintains the conda package. Use that package at your own risk. -## Installing with virtualenv +## Installing with Virtualenv Take the following steps to install TensorFlow with Virtualenv: 1. Start a terminal (a shell). You'll perform all subsequent steps in this shell. - 2. Install pip and virtualenv by issuing the following commands: + 2. Install pip and Virtualenv by issuing the following commands: 
 $ sudo easy_install pip
      $ pip install --upgrade virtualenv
- 3. Create a virtualenv environment by issuing a command of one + 3. Create a Virtualenv environment by issuing a command of one of the following formats: 
 $ virtualenv --system-site-packages targetDirectory # for Python 2.7
      $ virtualenv --system-site-packages -p python3 targetDirectory # for Python 3.n
- where targetDirectory identifies the top of the virtualenv tree. + where targetDirectory identifies the top of the Virtualenv tree. Our instructions assume that targetDirectory is `~/tensorflow`, but you may choose any directory. - 4. Activate the virtualenv environment by issuing one of the + 4. Activate the Virtualenv environment by issuing one of the following commands: 
$ source ~/tensorflow/bin/activate      # If using bash, sh, ksh, or zsh
@@ -93,7 +98,7 @@ Take the following steps to install TensorFlow with Virtualenv:
 
   7. Optional. If Step 6 failed (typically because you invoked a pip version
      lower than 8.1), install TensorFlow in the active
-     virtualenv environment by issuing a command of the following format:
+     Virtualenv environment by issuing a command of the following format:
 
      
 $ pip install --upgrade tfBinaryURL   # Python 2.7
      $ pip3 install --upgrade tfBinaryURL  # Python 3.n 

@@ -109,7 +114,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      
 $ pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl

+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl
If you encounter installation problems, see [Common Installation Problems](#common-installation-problems). @@ -121,8 +126,8 @@ After installing TensorFlow, [validate your installation](#ValidateYourInstallation) to confirm that the installation worked properly. -Note that you must activate the virtualenv environment each time you -use TensorFlow in a new shell. If the virtualenv environment is not +Note that you must activate the Virtualenv environment each time you +use TensorFlow in a new shell. If the Virtualenv environment is not currently active (that is, the prompt is not `(tensorflow)`, invoke one of the following commands: @@ -134,7 +139,7 @@ tensorflow environment is active: 
 (tensorflow)$
-When the virtualenv environment is active, you may run +When the Virtualenv environment is active, you may run TensorFlow programs from this shell. When you are done using TensorFlow, you may deactivate the @@ -230,7 +235,7 @@ take the following steps: issue the following command: 
 $ sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl If the preceding command fails, see [installation problems](#common-installation-problems). @@ -339,7 +344,7 @@ Take the following steps to install TensorFlow in an Anaconda environment: TensorFlow for Python 2.7: 
 (tensorflow)$ pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl @@ -353,11 +358,11 @@ To validate your TensorFlow installation, do the following: ### Prepare your environment -If you installed on native pip, virtualenv, or Anaconda, then +If you installed on native pip, Virtualenv, or Anaconda, then do the following: 1. Start a terminal. - 2. If you installed with virtualenv or Anaconda, activate your container. + 2. If you installed with Virtualenv or Anaconda, activate your container. 3. If you installed TensorFlow source code, navigate to any directory *except* one containing TensorFlow source code. @@ -512,7 +517,7 @@ This section documents the relevant values for Mac OS installations. 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py2-none-any.whl
@@ -520,7 +525,7 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py2-none-a 
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0dev-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.4.0rc1-py3-none-any.whl
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md index 6114496cd527ec59c228561333aee0cca47d76a7..6d0dcdcd4ae7f884b4afbd4803aebeb672a955d1 100644 --- a/tensorflow/docs_src/install/install_sources.md +++ b/tensorflow/docs_src/install/install_sources.md @@ -2,7 +2,7 @@ This guide explains how to build TensorFlow sources into a TensorFlow binary and how to install that TensorFlow binary. Note that we provide -well-tested, pre-built TensorFlow binaries for Linux, Mac, and Windows +well-tested, pre-built TensorFlow binaries for Ubuntu, macOS, and Windows systems. In addition, there are pre-built TensorFlow [docker images](https://hub.docker.com/r/tensorflow/tensorflow/). So, don't build a TensorFlow binary yourself unless you are very @@ -10,16 +10,22 @@ comfortable building complex packages from source and dealing with the inevitable aftermath should things not go exactly as documented. If the last paragraph didn't scare you off, welcome. This guide explains -how to build TensorFlow on the following operating systems: +how to build TensorFlow on 64-bit desktops and laptops running either of +the following operating systems: * Ubuntu -* Mac OS X +* macOS X -We don't officially support building TensorFlow on Windows; however, you may try -to build TensorFlow on Windows if you don't mind using the highly experimental -[Bazel on Windows](https://bazel.build/versions/master/docs/windows.html) -or -[TensorFlow CMake build](https://github.com/tensorflow/tensorflow/tree/r0.12/tensorflow/contrib/cmake). +Note: Some users have successfully built and installed TensorFlow from +sources on non-supported systems. Please remember that we do not fix +issues stemming from these attempts. + +We **do not support** building TensorFlow on Windows. That said, if you'd +like to try to build TensorFlow on Windows anyway, use either of the +following: + +* [Bazel on Windows](https://bazel.build/versions/master/docs/windows.html) +* [TensorFlow CMake build](https://github.com/tensorflow/tensorflow/tree/r0.12/tensorflow/contrib/cmake) ## Determine which TensorFlow to install @@ -40,7 +46,7 @@ install: software requirements described in one of the following documents: * @{$install_linux#NVIDIARequirements$Installing TensorFlow on Ubuntu} - * @{$install_mac#NVIDIARequirements$Installing TensorFlow on Mac OS} + * @{$install_mac#NVIDIARequirements$Installing TensorFlow on macOS} ## Clone the TensorFlow repository @@ -70,7 +76,7 @@ issue the following command: Next, you must prepare your environment for [Linux](#PrepareLinux) or -[Mac OS](#PrepareMac) +[macOS](#PrepareMac) @@ -157,7 +163,7 @@ After preparing the environment, you must now -## Prepare environment for Mac OS +## Prepare environment for macOS Before building TensorFlow, you must install the following on your system: @@ -238,8 +244,8 @@ One of the questions that `configure` will ask is as follows: Please specify optimization flags to use during compilation when bazel option "--config=opt" is specified [Default is -march=native] -This question refers to a later phase in which you'll use bazel to -[build the pip package](#build-the-pip-package). We recommend +This question refers to a later phase in which you'll use bazel to +[build the pip package](#build-the-pip-package). We recommend accepting the default (`-march=native`), which will optimize the generated code for your local machine's CPU type. However, if you are building TensorFlow on one CPU type but will run TensorFlow on @@ -288,7 +294,7 @@ Please specify a list of comma-separated Cuda compute capabilities you want to b You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus. Please note that each additional compute capability significantly increases your build time and binary size. [Default is: "3.5,5.2"]: 3.0 -Do you wish to build TensorFlow with MPI support? [y/N] +Do you wish to build TensorFlow with MPI support? [y/N] MPI support will not be enabled for TensorFlow Configuration finished @@ -349,10 +355,10 @@ Invoke `pip install` to install that pip package. The filename of the `.whl` file depends on your platform. For example, the following command will install the pip package -for TensorFlow 1.4.0dev on Linux: +for TensorFlow 1.4.0rc1 on Linux: 
-$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0dev-py2-none-any.whl
+$ sudo pip install /tmp/tensorflow_pkg/tensorflow-1.4.0rc1-py2-none-any.whl
## Validate your installation @@ -441,8 +447,8 @@ Stack Overflow and specify the `tensorflow` tag. **Linux**
Example Data Set Demonstrates How To...
Example Demonstrates How To...
linear_regression.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use the @{tf.estimator.LinearRegressor} Estimator to train a regression model on numeric data.
linear_regression_categorical.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use the @{tf.estimator.LinearRegressor} Estimator to train a regression model on categorical data.
dnn_regression.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use the @{tf.estimator.DNNRegressor} Estimator to train a regression model on discrete data with a deep neural network.
custom_regression.py[imports85](https://archive.ics.uci.edu/ml/datasets/automobile) Use @{tf.estimator.Estimator} to train a customized dnn regression model.
- - + + @@ -454,19 +460,19 @@ Stack Overflow and specify the `tensorflow` tag. **Mac**
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.4.0rc0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.568
tensorflow-1.4.0rc1CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.4.0rc1GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.568
tensorflow-1.2.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.5N/AN/A
tensorflow_gpu-1.2.0GPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.55.18
tensorflow-1.1.0CPU2.7, 3.3-3.6GCC 4.8Bazel 0.4.2N/AN/A
- - - - - - + + + + + +
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
ttensorflow-1.2.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
ttensorflow-1.1.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
ttensorflow_gpu-1.1.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
ttensorflow-1.0.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
ttensorflow_gpu-1.0.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
tensorflow-1.4.0rc1CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.2.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.5N/AN/A
tensorflow-1.1.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
tensorflow_gpu-1.1.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
tensorflow-1.0.0CPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.2N/AN/A
tensorflow_gpu-1.0.0GPU2.7, 3.3-3.6Clang from xcodeBazel 0.4.25.18
**Windows** - - + + diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md index f0d580d803404ef0508e7841a6040e4656f33711..4098ee5b2e51521c9c77dadc9dbf0eb6f6c78235 100644 --- a/tensorflow/docs_src/install/install_windows.md +++ b/tensorflow/docs_src/install/install_windows.md @@ -1,6 +1,13 @@ # Installing TensorFlow on Windows -This guide explains how to install TensorFlow on Windows. +This guide explains how to install TensorFlow on Windows. Although these +instructions might also work on other Windows variants, we have only +tested (and we only support) these instructions on machines meeting the +following requirements: + + * 64-bit, x86 desktops or laptops + * Windows 7 or later + ## Determine which TensorFlow to install diff --git a/tensorflow/docs_src/mobile/android_build.md b/tensorflow/docs_src/mobile/android_build.md new file mode 100644 index 0000000000000000000000000000000000000000..030cd0d051103e0d4bf903663d6fb7300c884b18 --- /dev/null +++ b/tensorflow/docs_src/mobile/android_build.md @@ -0,0 +1,176 @@ +# Building TensorFlow on Android + +To get you started working with TensorFlow on Android, we'll walk through two +ways to build our TensorFlow mobile demos and deploying them on an Android +device. The first is Android Studio, which lets you build and deploy in an +IDE. The second is building with Bazel and deploying with ADB on the command +line. + +Why choose one or the other of these methods? + +The simplest way to use TensorFlow on Android is to use Android Studio. If you +aren't planning to customize your TensorFlow build at all, or if you want to use +Android Studio's editor and other features to build an app and just want to add +TensorFlow to it, we recommend using Android Studio. + +If you are using custom ops, or have some other reason to build TensorFlow from +scratch, scroll down and see our instructions +for [building the demo with Bazel](#build_the_demo_using_bazel). + +## Build the demo using Android Studio + +**Prerequisites** + +If you haven't already, do the following two things: + +- Install [Android Studio](https://developer.android.com/studio/index.html), + following the instructions on their website. + +- Clone the TensorFlow repository from Github: + + git clone https://github.com/tensorflow/tensorflow + +**Building** + +1. Open Android Studio, and from the Welcome screen, select **Open an existing + Android Studio project**. + +2. From the **Open File or Project** window that appears, navigate to and select + the `tensorflow/examples/android` directory from wherever you cloned the + TensorFlow Github repo. Click OK. + + If it asks you to do a Gradle Sync, click OK. + + You may also need to install various platforms and tools, if you get + errors like "Failed to find target with hash string 'android-23' and similar. + +3. Open the `build.gradle` file (you can go to **1:Project** in the side panel + and find it under the **Gradle Scripts** zippy under **Android**). Look for + the `nativeBuildSystem` variable and set it to `none` if it isn't already: + + // set to 'bazel', 'cmake', 'makefile', 'none' + def nativeBuildSystem = 'none' + +4. Click the Run button (the green arrow) or use **Run -> Run 'android'** from the top menu. + + If it asks you to use Instant Run, click **Proceed Without Instant Run**. + + Also, you need to have an Android device plugged in with developer options + enabled at this + point. See [here](https://developer.android.com/studio/run/device.html) for + more details on setting up developer devices. + +This installs three apps on your phone that are all part of the TensorFlow +Demo. See [Android Sample Apps](#android_sample_apps) for more information about +them. + +## Adding TensorFlow to your apps using Android Studio + +To add TensorFlow to your own apps on Android, the simplest way is to add the +following lines to your Gradle build file: + + allprojects { + repositories { + jcenter() + } + } + + dependencies { + compile 'org.tensorflow:tensorflow-android:+' + } + +This automatically downloads the latest stable version of TensorFlow as an AAR +and installs it in your project. + +## Build the demo using Bazel + +Another way to use TensorFlow on Android is to build an APK +using [Bazel](https://bazel.build/) and load it onto your device +using [ADB](https://developer.android.com/studio/command-line/adb.html). This +requires some knowledge of build systems and Android developer tools, but we'll +guide you through the basics here. + +- First, follow our instructions for @{$install/install_sources$installing from + sources}. This will also guide you through installing Bazel and cloning the + TensorFlow code. + +- Download the Android [SDK](https://developer.android.com/studio/index.html) + and [NDK](https://developer.android.com/ndk/downloads/index.html) if you do + not already have them. You need at least version 12b of the NDK, and 23 of the + SDK. + +- In your copy of the TensorFlow source, update the + [WORKSPACE](https://github.com/tensorflow/tensorflow/blob/master/WORKSPACE) + file with the location of your SDK and NDK, where it says <PATH_TO_NDK> + and <PATH_TO_SDK>. + +- Run Bazel to build the demo APK: + + bazel build -c opt //tensorflow/examples/android:tensorflow_demo + +- Use [ADB](https://developer.android.com/studio/command-line/adb.html#move) to + install the APK onto your device: + + adb install -r bazel-bin/tensorflow/examples/android/tensorflow_demo.apk + +Note: In general when compiling for Android with Bazel you need +`--config=android` on the Bazel command line, though in this case this +particular example is Android-only, so you don't need it here. + +This installs three apps on your phone that are all part of the TensorFlow +Demo. See [Android Sample Apps](#android_sample_apps) for more information about +them. + +## Android Sample Apps + +The +[Android example code](https://www.tensorflow.org/code/tensorflow/examples/android/) is +a single project that builds and installs three sample apps which all use the +same underlying code. The sample apps all take video input from a phone's +camera: + +- **TF Classify** uses the Inception v3 model to label the objects it’s pointed + at with classes from Imagenet. There are only 1,000 categories in Imagenet, + which misses most everyday objects and includes many things you’re unlikely to + encounter often in real life, so the results can often be quite amusing. For + example there’s no ‘person’ category, so instead it will often guess things it + does know that are often associated with pictures of people, like a seat belt + or an oxygen mask. If you do want to customize this example to recognize + objects you care about, you can use + the + [TensorFlow for Poets codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) as + an example for how to train a model based on your own data. + +- **TF Detect** uses a multibox model to try to draw bounding boxes around the + locations of people in the camera. These boxes are annotated with the + confidence for each detection result. Results will not be perfect, as this + kind of object detection is still an active research topic. The demo also + includes optical tracking for when objects move between frames, which runs + more frequently than the TensorFlow inference. This improves the user + experience since the apparent frame rate is faster, but it also gives the + ability to estimate which boxes refer to the same object between frames, which + is important for counting objects over time. + +- **TF Stylize** implements a real-time style transfer algorithm on the camera + feed. You can select which styles to use and mix between them using the + palette at the bottom of the screen, and also switch out the resolution of the + processing to go higher or lower rez. + +When you build and install the demo, you'll see three app icons on your phone, +one for each of the demos. Tapping on them should open up the app and let you +explore what they do. You can enable profiling statistics on-screen by tapping +the volume up button while they’re running. + +### Android Inference Library + +Because Android apps need to be written in Java, and core TensorFlow is in C++, +TensorFlow has a JNI library to interface between the two. Its interface is aimed +only at inference, so it provides the ability to load a graph, set up inputs, +and run the model to calculate particular outputs. You can see the full +documentation for the minimal set of methods in +[TensorFlowInferenceInterface.java](https://www.tensorflow.org/code/tensorflow/contrib/android/java/org/tensorflow/contrib/android/TensorFlowInferenceInterface.java) + +The demos applications use this interface, so they’re a good place to look for +example usage. You can download prebuilt binary jars +at +[ci.tensorflow.org](https://ci.tensorflow.org/view/Nightly/job/nightly-android/). diff --git a/tensorflow/docs_src/mobile/index.md b/tensorflow/docs_src/mobile/index.md new file mode 100644 index 0000000000000000000000000000000000000000..a6f1422f6f170fee1a24fa12f62fc03d60632666 --- /dev/null +++ b/tensorflow/docs_src/mobile/index.md @@ -0,0 +1,238 @@ +# Building Mobile Apps with TensorFlow + +TensorFlow was designed from the ground up to be a good deep learning solution +for mobile platforms like Android and iOS. This guide is to help you understand +how to integrate TensorFlow into your mobile apps effectively and efficiently. + +## About this Guide + +This guide is aimed at developers who have a TensorFlow model that’s +successfully working in a desktop environment, and who want to integrate it into +a mobile application. Here are the main challenges you’ll face during that +process: + +- Understanding how to use Tensorflow for mobile. +- Building TensorFlow for your platform. +- Integrating the TensorFlow library into your application. +- Preparing your model file for mobile deployment. +- Optimizing for latency, RAM usage, model file size, and binary size. + +## Why run TensorFlow on mobile? + +Traditionally, deep learning has been associated with data centers and giant +clusters of high-powered GPU machines. However, it can be very expensive and +time-consuming to send all of the data a device has access to across a network +connection. Running on mobile makes it possible to deliver very interactive +applications in a way that’s not possible when you have to wait for a network +round trip. + +Here are some common use cases for on-device deep learning: + +### Speech Recognition + +There are a lot of interesting applications that can be built with a +speech-driven interface, and many of these require on-device processing. Most of +the time a user isn’t giving commands, and so streaming audio continuously to a +remote server would be a waste of bandwidth, since it would mostly be silence or +background noises. To solve this problem it’s common to have a small neural +network running on-device @{$tutorials/audio_recognition$listening out for a +particular keyword}. Once that keyword has been spotted, the rest of the +conversation can be transmitted over to the server for further processing if +more computing power is needed. + +### Image Recognition + +It can be very useful for a mobile app to be able to make sense of a camera +image. If your users are taking photos, recognizing what’s in them can help your +camera apps apply appropriate filters, or label the photos so they’re easily +findable. It’s important for embedded applications too, since you can use image +sensors to detect all sorts of interesting conditions, whether it’s spotting +endangered animals in the wild +or +[reporting how late your train is running](https://svds.com/tensorflow-image-recognition-raspberry-pi/). + +TensorFlow comes with several examples of recognizing the types of objects +inside images along with a variety of different pre-trained models, and they can +all be run on mobile devices. You can try out +our +[Tensorflow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/index.html#0) and +[Tensorflow for Poets 2: Optimize for Mobile](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/index.html#0) codelabs to +see how to take a pretrained model and run some very fast and lightweight +training to teach it to recognize specific objects, and then optimize it to +run on mobile. + +### Object Localization + +Sometimes it’s important to know where objects are in an image as well as what +they are. There are lots of augmented reality use cases that could benefit a +mobile app, such as guiding users to the right component when offering them +help fixing their wireless network or providing informative overlays on top of +landscape features. Embedded applications often need to count objects that are +passing by them, whether it’s pests in a field of crops, or people, cars and +bikes going past a street lamp. + +TensorFlow offers a pretrained model for drawing bounding boxes around people +detected in images, together with tracking code to follow them over time. The +tracking is especially important for applications where you’re trying to count +how many objects are present over time, since it gives you a good idea when a +new object enters or leaves the scene. We have some sample code for this +available for Android [on +Github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/android), +and also a [more general object detection +model](https://github.com/tensorflow/models/tree/master/object_detection/README.md) +available as well. + +### Gesture Recognition + +It can be useful to be able to control applications with hand or other +gestures, either recognized from images or through analyzing accelerometer +sensor data. Creating those models is beyond the scope of this guide, but +TensorFlow is an effective way of deploying them. + +### Optical Character Recognition + +Google Translate’s live camera view is a great example of how effective +interactive on-device detection of text can be. + +
+ +
+ +There are multiple steps involved in recognizing text in images. You first have +to identify the areas where the text is present, which is a variation on the +object localization problem, and can be solved with similar techniques. Once you +have an area of text, you then need to interpret it as letters, and then use a +language model to help guess what words they represent. The simplest way to +estimate what letters are present is to segment the line of text into individual +letters, and then apply a simple neural network to the bounding box of each. You +can get good results with the kind of models used for MNIST, which you can find +in TensorFlow’s tutorials, though you may want a higher-resolution input. A +more advanced alternative is to use an LSTM model to process a whole line of +text at once, with the model itself handling the segmentation into different +characters. + +### Translation + +Translating from one language to another quickly and accurately, even if you +don’t have a network connection, is an important use case. Deep networks are +very effective at this sort of task, and you can find descriptions of a lot of +different models in the literature. Often these are sequence-to-sequence +recurrent models where you’re able to run a single graph to do the whole +translation, without needing to run separate parsing stages. + +### Text Classification + +If you want to suggest relevant prompts to users based on what they’re typing or +reading, it can be very useful to understand the meaning of the text. This is +where text classification comes in. Text classification is an umbrella term +that covers everything from sentiment analysis to topic discovery. You’re likely +to have your own categories or labels that you want to apply, so the best place +to start is with an example +like +[Skip-Thoughts](https://github.com/tensorflow/models/tree/master/skip_thoughts/), +and then train on your own examples. + +### Voice Synthesis + +A synthesized voice can be a great way of giving users feedback or aiding +accessibility, and recent advances such as +[WaveNet](https://deepmind.com/blog/wavenet-generative-model-raw-audio/) show +that deep learning can offer very natural-sounding speech. + +## How does it fit with the cloud? + +These examples of use cases give an idea of how on-device networks can +complement cloud services. Cloud has a great deal of computing power in a +controlled environment, but running on devices can offer higher interactivity. +In situations where the cloud is unavailable, or your cloud capacity is limited, +you can provide an offline experience, or reduce cloud workload by processing +easy cases on device. + +Doing on-device computation can also signal when it's time to switch to working +on the cloud. A good example of this is hotword detection in speech. Since +devices are able to constantly listen out for the keywords, this then triggers a +lot of traffic to cloud-based speech recognition once one is recognised. Without +the on-device component, the whole application wouldn’t be feasible, and this +pattern exists across several other applications as well. Recognizing that some +sensor input is interesting enough for further processing makes a lot of +interesting products possible. + +## What hardware and software should you have? + +TensorFlow runs on Ubuntu Linux, Windows 10, and OS X. For a list of all +supported operating systems and instructions to install TensorFlow, see +@{$install$Installing Tensorflow}. + +Some of the scripts in this guide require you to compile TensorFlow from source, +so you’ll need more than just `pip install` to work through all the sample code. + +To try out the mobile examples, you’ll need a device set up for development, +using +either [Android Studio](https://developer.android.com/studio/install.html), +or [XCode](https://developer.apple.com/xcode/) if you're developing for iOS. + +## What should you do before you get started? + +Before thinking about how to get your solution on mobile: + +1. Determine whether your problem is solvable by mobile machine learning +2. Create a labelled dataset to define your problem +3. Pick an effective model for the problem + +We'll discuss these in more detail below. + +### Is your problem solvable by mobile machine learning? + +Once you have an idea of the problem you want to solve, you need to make a plan +of how to build your solution. The most important first step is making sure that +your problem is actually solvable, and the best way to do that is to mock it up +using humans in the loop. + +For example, if you want to drive a robot toy car using voice commands, try +recording some audio from the device and listen back to it to see if you can +make sense of what’s being said. Often you’ll find there are problems in the +capture process, such as the motor drowning out speech or not being able to hear +at a distance, and you should tackle these problems before investing in the +modeling process. + +Another example would be giving photos taken from your app to people see if they +can classify what’s in them, in the way you’re looking for. If they can’t do +that (for example, trying to estimate calories in food from photos may be +impossible because all white soups look the same), then you’ll need to redesign +your experience to cope with that. A good rule of thumb is that if a human can’t +handle the task then it will be difficult to train a computer to do better. + +### Create a labelled dataset + +After you’ve solved any fundamental issues with your use case, you need to +create a labeled dataset to define what problem you’re trying to solve. This +step is extremely important, moreso than picking which model to use. You want it +to be as representative as possible of your actual use case, since the model +will only be effective at the task you teach it. It’s also worth investing in +tools to make labeling the data as efficient and accurate as possible. For +example, if you’re able to switch from having to click a button on a web +interface to simple keyboard shortcuts, you may be able to speed up the +generation process a lot. You should also start by doing the initial labeling +yourself, so you can learn about the difficulties and likely errors, and +possibly change your labeling or data capture process to avoid them. Once you +and your team are able to consistently label examples (that is once you +generally agree on the same labels for most examples), you can then try and +capture your knowledge in a manual and teach external raters how to run the same +process. + +### Pick an effective model + +The next step is to pick an effective model to use. You might be able to avoid +training a model from scratch if someone else has already implemented a model +similar to what you need; we have a repository of models implemented in +TensorFlow [on Github](https://github.com/tensorflow/models) that you can look +through. Lean towards the simplest model you can find, and try to get started as +soon as you have even a small amount of labelled data, since you’ll get the best +results when you’re able to iterate quickly. The shorter the time it takes to +try training a model and running it in s real application, the better overall +results you’ll see. It’s common for an algorithm to get great training accuracy +numbers but then fail to be useful within a real application because there’s a +mismatch between the dataset and real usage. Prototype end-to-end usage as soon +as possible to create a consistent user experience. diff --git a/tensorflow/docs_src/mobile/ios_build.md b/tensorflow/docs_src/mobile/ios_build.md new file mode 100644 index 0000000000000000000000000000000000000000..2e6d3bf90e739aa3dce2a8dfb2568383b68b0282 --- /dev/null +++ b/tensorflow/docs_src/mobile/ios_build.md @@ -0,0 +1,107 @@ +# Building TensorFlow on iOS + +## Using CocoaPods + +The simplest way to get started with TensorFlow on iOS is using the CocoaPods +package management system. You can add the `TensorFlow-experimental` pod to your +Podfile, which installs a universal binary framework. This makes it easy to get +started but has the disadvantage of being hard to customize, which is important +in case you want to shrink your binary size. If you do need the ability to +customize your libraries, see later sections on how to do that. + +## Creating your own app + +If you'd like to add TensorFlow capabilities to your own app, do the following: + +- Create your own app or load your already-created app in XCode. + +- Add a file named Podfile at the project root directory with the following content: + + target 'YourProjectName' + pod 'TensorFlow-experimental' + +- Run `pod install` to download and install the `TensorFlow-experimental` pod. + +- Open `YourProjectName.xcworkspace` and add your code. + +- In your app's **Build Settings**, make sure to add `$(inherited)` to the + **Other Linker Flags**, and **Header Search Paths** sections. + +## Running the Samples + +You'll need Xcode 7.3 or later to run our iOS samples. + +There are currently three examples: simple, benchmark, and camera. For now, you +can download the sample code by cloning the main tensorflow repository (we are +planning to make the samples available as a separate repository later). + +From the root of the tensorflow folder, download [Inception +v1](https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip), +and extract the label and graph files into the data folders inside both the +simple and camera examples using these steps: + + mkdir -p ~/graphs + curl -o ~/graphs/inception5h.zip \ + https://storage.googleapis.com/download.tensorflow.org/models/inception5h.zip \ + && unzip ~/graphs/inception5h.zip -d ~/graphs/inception5h + cp ~/graphs/inception5h/* tensorflow/examples/ios/benchmark/data/ + cp ~/graphs/inception5h/* tensorflow/examples/ios/camera/data/ + cp ~/graphs/inception5h/* tensorflow/examples/ios/simple/data/ + +Change into one of the sample directories, download the +[Tensorflow-experimental](https://cocoapods.org/pods/TensorFlow-experimental) +pod, and open the Xcode workspace. Note that installing the pod can take a long +time since it is big (~450MB). If you want to run the simple example, then: + + cd tensorflow/examples/ios/simple + pod install + open tf_simple_example.xcworkspace # note .xcworkspace, not .xcodeproj + # this is created by pod install + +Run the simple app in the XCode simulator. You should see a single-screen app +with a **Run Model** button. Tap that, and you should see some debug output +appear below indicating that the example Grace Hopper image in directory data +has been analyzed, with a military uniform recognized. + +Run the other samples using the same process. The camera example requires a real +device connected. Once you build and run that, you should get a live camera view +that you can point at objects to get real-time recognition results. + +### iOS Example details + +There are three demo applications for iOS, all defined in Xcode projects inside +[tensorflow/examples/ios](https://www.tensorflow.org/code/tensorflow/examples/ios/). + +- **Simple**: This is a minimal example showing how to load and run a TensorFlow + model in as few lines as possible. It just consists of a single view with a + button that executes the model loading and inference when its pressed. + +- **Camera**: This is very similar to the Android TF Classify demo. It loads + Inception v3 and outputs its best label estimate for what’s in the live camera + view. As with the Android version, you can train your own custom model using + TensorFlow for Poets and drop it into this example with minimal code changes. + +- **Benchmark**: is quite close to Simple, but it runs the graph repeatedly and + outputs similar statistics to the benchmark tool on Android. + + +### Troubleshooting + +- Make sure you use the TensorFlow-experimental pod (and not TensorFlow). + +- The TensorFlow-experimental pod is current about ~450MB. The reason it is so + big is because we are bundling multiple platforms, and the pod includes all + TensorFlow functionality (e.g. operations). The final app size after build is + substantially smaller though (~25MB). Working with the complete pod is + convenient during development, but see below section on how you can build your + own custom TensorFlow library to reduce the size. + +## Building the TensorFlow iOS libraries from source + +While Cocapods is the quickest and easiest way of getting started, you sometimes +need more flexibility to determine which parts of TensorFlow your app should be +shipped with. For such cases, you can build the iOS libraries from the +sources. [This +guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/examples/ios#building-the-tensorflow-ios-libraries-from-source) +contains detailed instructions on how to do that. + diff --git a/tensorflow/docs_src/mobile/leftnav_files b/tensorflow/docs_src/mobile/leftnav_files new file mode 100644 index 0000000000000000000000000000000000000000..347c07d2330fb0da7e5c9f287ddba16524e4ec34 --- /dev/null +++ b/tensorflow/docs_src/mobile/leftnav_files @@ -0,0 +1,8 @@ +### TensorFlow for Mobile +index.md +android_build.md +ios_build.md +#raspi_build.md until this section gets rewritten, or TFLite takes over +linking_libs.md +prepare_models.md +optimizing.md diff --git a/tensorflow/docs_src/mobile/linking_libs.md b/tensorflow/docs_src/mobile/linking_libs.md new file mode 100644 index 0000000000000000000000000000000000000000..2a0a77c92d309edb654486d2cd841b72a35840d0 --- /dev/null +++ b/tensorflow/docs_src/mobile/linking_libs.md @@ -0,0 +1,243 @@ +# Integrating TensorFlow libraries + +Once you have made some progress on a model that addresses the problem you’re +trying to solve, it’s important to test it out inside your application +immediately. There are often unexpected differences between your training data +and what users actually encounter in the real world, and getting a clear picture +of the gap as soon as possible improves the product experience. + +This page talks about how to integrate the TensorFlow libraries into your own +mobile applications, once you have already successfully built and deployed the +TensorFlow mobile demo apps. + +## Linking the library + +After you've managed to build the examples, you'll probably want to call +TensorFlow from one of your existing applications. The very easiest way to do +this is to use the Pod installation steps described +@{$mobile/ios_build#using_cocoapods$here}, but if you want to build TensorFlow +from source (for example to customize which operators are included) you'll need +to break out TensorFlow as a framework, include the right header files, and link +against the built libraries and dependencies. + +### Android + +For Android, you just need to link in a Java library contained in a JAR file +called `libandroid_tensorflow_inference_java.jar`. There are three ways to +include this functionality in your program: + +1. Include the jcenter AAR which contains it, as in this + [example app](https://github.com/googlecodelabs/tensorflow-for-poets-2/blob/master/android/build.gradle#L59-L65) + +2. Download the nightly precompiled version from +[ci.tensorflow.org](http://ci.tensorflow.org/view/Nightly/job/nightly-android/lastSuccessfulBuild/artifact/out/). + +3. Build the JAR file yourself using the instructions [in our Android Github repo](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/android) + +### iOS + +Pulling in the TensorFlow libraries on iOS is a little more complicated. Here is +a checklist of what you’ll need to do to your iOS app: + +- Link against tensorflow/contrib/makefile/gen/lib/libtensorflow-core.a, usually + by adding `-L/your/path/tensorflow/contrib/makefile/gen/lib/` and + `-ltensorflow-core` to your linker flags. + +- Link against the generated protobuf libraries by adding + `-L/your/path/tensorflow/contrib/makefile/gen/protobuf_ios/lib` and + `-lprotobuf` and `-lprotobuf-lite` to your command line. + +- For the include paths, you need the root of your TensorFlow source folder as + the first entry, followed by + `tensorflow/contrib/makefile/downloads/protobuf/src`, + `tensorflow/contrib/makefile/downloads`, + `tensorflow/contrib/makefile/downloads/eigen`, and + `tensorflow/contrib/makefile/gen/proto`. + +- Make sure your binary is built with `-force_load` (or the equivalent on your + platform), aimed at the TensorFlow library to ensure that it’s linked + correctly. More detail on why this is necessary can be found in the next + section, [Global constructor magic](#global_constructor_magic). On Linux-like + platforms, you’ll need different flags, more like + `-Wl,--allow-multiple-definition -Wl,--whole-archive`. + +You’ll also need to link in the Accelerator framework, since this is used to +speed up some of the operations. + +## Global constructor magic + +One of the subtlest problems you may run up against is the “No session factory +registered for the given session options” error when trying to call TensorFlow +from your own application. To understand why this is happening and how to fix +it, you need to know a bit about the architecture of TensorFlow. + +The framework is designed to be very modular, with a thin core and a large +number of specific objects that are independent and can be mixed and matched as +needed. To enable this, the coding pattern in C++ had to let modules easily +notify the framework about the services they offer, without requiring a central +list that has to be updated separately from each implementation. It also had to +allow separate libraries to add their own implementations without needing a +recompile of the core. + +To achieve this capability, TensorFlow uses a registration pattern in a lot of +places. In the code, it looks like this: + + class MulKernel : OpKernel { + Status Compute(OpKernelContext* context) { … } + }; + REGISTER_KERNEL(MulKernel, “Mul”); + +This would be in a standalone `.cc` file linked into your application, either +as part of the main set of kernels or as a separate custom library. The magic +part is that the `REGISTER_KERNEL()` macro is able to inform the core of +TensorFlow that it has an implementation of the Mul operation, so that it can be +called in any graphs that require it. + +From a programming point of view, this setup is very convenient. The +implementation and registration code live in the same file, and adding new +implementations is as simple as compiling and linking it in. The difficult part +comes from the way that the `REGISTER_KERNEL()` macro is implemented. C++ +doesn’t offer a good mechanism for doing this sort of registration, so we have +to resort to some tricky code. Under the hood, the macro is implemented so that +it produces something like this: + + class RegisterMul { + public: + RegisterMul() { + global_kernel_registry()->Register(“Mul”, [](){ + return new MulKernel() + }); + } + }; + RegisterMul g_register_mul; + +This sets up a class `RegisterMul` with a constructor that tells the global +kernel registry what function to call when somebody asks it how to create a +“Mul” kernel. Then there’s a global object of that class, and so the constructor +should be called at the start of any program. + +While this may sound sensible, the unfortunate part is that the global object +that’s defined is not used by any other code, so linkers not designed with this +in mind will decide that it can be deleted. As a result, the constructor is +never called, and the class is never registered. All sorts of modules use this +pattern in TensorFlow, and it happens that `Session` implementations are the +first to be looked for when the code is run, which is why it shows up as the +characteristic error when this problem occurs. + +The solution is to force the linker to not strip any code from the library, even +if it believes it’s unused. On iOS, this step can be accomplished with the +`-force_load` flag, specifying a library path, and on Linux you need +`--whole-archive`. These persuade the linker to not be as aggressive about +stripping, and should retain the globals. + +The actual implementation of the various `REGISTER_*` macros is a bit more +complicated in practice, but they all suffer the same underlying problem. If +you’re interested in how they work, [op_kernel.h](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/op_kernel.h#L1091) +is a good place to start investigating. + +## Protobuf problems + +TensorFlow relies on +the [Protocol Buffer](https://developers.google.com/protocol-buffers/) library, +commonly known as protobuf. This library takes definitions of data structures +and produces serialization and access code for them in a variety of +languages. The tricky part is that this generated code needs to be linked +against shared libraries for the exact same version of the framework that was +used for the generator. This can be an issue when `protoc`, the tool used to +generate the code, is from a different version of protobuf than the libraries in +the standard linking and include paths. For example, you might be using a copy +of `protoc` that was built locally in `~/projects/protobuf-3.0.1.a`, but you have +libraries installed at `/usr/local/lib` and `/usr/local/include` that are from +3.0.0. + +The symptoms of this issue are errors during the compilation or linking phases +with protobufs. Usually, the build tools take care of this, but if you’re using +the makefile, make sure you’re building the protobuf library locally and using +it, as shown in [this Makefile](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/Makefile#L18). + +Another situation that can cause problems is when protobuf headers and source +files need to be generated as part of the build process. This process makes +building more complex, since the first phase has to be a pass over the protobuf +definitions to create all the needed code files, and only after that can you go +ahead and do a build of the library code. + +### Multiple versions of protobufs in the same app + +Protobufs generate headers that are needed as part of the C++ interface to the +overall TensorFlow library. This complicates using the library as a standalone +framework. + +If your application is already using version 1 of the protocol buffers library, +you may have trouble integrating TensorFlow because it requires version 2. If +you just try to link both versions into the same binary, you’ll see linking +errors because some of the symbols clash. To solve this particular problem, we +have an experimental script at [rename_protobuf.sh](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/rename_protobuf.sh). + +You need to run this as part of the makefile build, after you’ve downloaded all +the dependencies: + + tensorflow/contrib/makefile/download_dependencies.sh + tensorflow/contrib/makefile/rename_protobuf.sh + +## Calling the TensorFlow API + +Once you have the framework available, you then need to call into it. The usual +pattern is that you first load your model, which represents a preset set of +numeric computations, and then you run inputs through that model (for example, +images from a camera) and receive outputs (for example, predicted labels). + +On Android, we provide the Java Inference Library that is focused on just this +use case, while on iOS and Raspberry Pi you call directly into the C++ API. + +### Android + +Here’s what a typical Inference Library sequence looks like on Android: + + // Load the model from disk. + TensorFlowInferenceInterface inferenceInterface = + new TensorFlowInferenceInterface(assetManager, modelFilename); + + // Copy the input data into TensorFlow. + inferenceInterface.feed(inputName, floatValues, 1, inputSize, inputSize, 3); + + // Run the inference call. + inferenceInterface.run(outputNames, logStats); + + // Copy the output Tensor back into the output array. + inferenceInterface.fetch(outputName, outputs); + +You can find the source of this code in the [Android examples](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/android/src/org/tensorflow/demo/TensorFlowImageClassifier.java#L107). + +### iOS and Raspberry Pi + +Here’s the equivalent code for iOS and Raspberry Pi: + + // Load the model. + PortableReadFileToProto(file_path, &tensorflow_graph); + + // Create a session from the model. + tensorflow::Status s = session->Create(tensorflow_graph); + if (!s.ok()) { + LOG(FATAL) << "Could not create TensorFlow Graph: " << s; + } + + // Run the model. + std::string input_layer = "input"; + std::string output_layer = "output"; + std::vector outputs; + tensorflow::Status run_status = session->Run({{input_layer, image_tensor}}, + {output_layer}, {}, &outputs); + if (!run_status.ok()) { + LOG(FATAL) << "Running model failed: " << run_status; + } + + // Access the output data. + tensorflow::Tensor* output = &outputs[0]; + +This is all based on the +[iOS sample code](https://www.tensorflow.org/code/tensorflow/examples/ios/simple/RunModelViewController.mm), +but there’s nothing iOS-specific; the same code should be usable on any platform +that supports C++. + +You can also find specific examples for Raspberry Pi +[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/pi_examples/label_image/label_image.cc). diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md new file mode 100644 index 0000000000000000000000000000000000000000..1da8be5689c9ac4f5d0bfdd364c8da653618f654 --- /dev/null +++ b/tensorflow/docs_src/mobile/optimizing.md @@ -0,0 +1,497 @@ +# Optimizing for mobile + +There are some special issues that you have to deal with when you’re trying to +ship on mobile or embedded devices, and you’ll need to think about these as +you’re developing your model. + +These issues are: + +- Model and Binary Size +- App speed and model loading speed +- Performance and threading + +We'll discuss a few of these below. + +## What are the minimum device requirements for TensorFlow? + +You need at least one megabyte of program memory and several megabytes of RAM to +run the base TensorFlow runtime, so it’s not suitable for DSPs or +microcontrollers. Other than those, the biggest constraint is usually the +calculation speed of the device, and whether you can run the model you need for +your application with a low enough latency. You can use the benchmarking tools +in [How to Profile your Model](#how_to_profile_your_model) to get an idea of how +many FLOPs are required for a model, and then use that to make rule-of-thumb +estimates of how fast they will run on different devices. For example, a modern +smartphone might be able to run 10 GFLOPs per second, so the best you could hope +for from a 5 GFLOP model is two frames per second, though you may do worse +depending on what the exact computation patterns are. + +This model dependence means that it’s possible to run TensorFlow even on very +old or constrained phones, as long as you optimize your network to fit within +the latency budget and possibly within limited RAM too. For memory usage, you +mostly need to make sure that the intermediate buffers that TensorFlow creates +aren’t too large, which you can examine in the benchmark output too. + +## Speed + +One of the highest priorities of most model deployments is figuring out how to +run the inference fast enough to give a good user experience. The first place to +start is by looking at the total number of floating point operations that are +required to execute the graph. You can get a very rough estimate of this by +using the `benchmark_model` tool: + + bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \ + bazel-bin/tensorflow/tools/benchmark/benchmark_model \ + --graph=/tmp/inception_graph.pb --input_layer="Mul:0" \ + --input_layer_shape="1,299,299,3" --input_layer_type="float" \ + --output_layer="softmax:0" --show_run_order=false --show_time=false \ + --show_memory=false --show_summary=true --show_flops=true --logtostderr + +This should show you an estimate of how many operations are needed to run the +graph. You can then use that information to figure out how feasible your model +is to run on the devices you’re targeting. For an example, a high-end phone from +2016 might be able to do 20 billion FLOPs per second, so the best speed you +could hope for from a model that requires 10 billion FLOPs is around 500ms. On a +device like the Raspberry Pi 3 that can do about 5 billion FLOPs, you may only +get one inference every two seconds. + +Having this estimate helps you plan for what you’ll be able to realistically +achieve on a device. If the model is using too many ops, then there are a lot of +opportunities to optimize the architecture to reduce that number. + +Advanced techniques include [SqueezeNet](https://arxiv.org/abs/1602.07360) +and [MobileNet](https://arxiv.org/abs/1704.04861), which are architectures +designed to produce models for mobile -- lean and fast but with a small accuracy +cost. You can also just look at alternative models, even older ones, which may +be smaller. For example, Inception v1 only has around 7 million parameters, +compared to Inception v3’s 24 million, and requires only 3 billion FLOPs rather +than 9 billion for v3. + +## Model Size + +Models that run on a device need to be stored somewhere on the device, and very +large neural networks can be hundreds of megabytes. Most users are reluctant to +download very large app bundles from app stores, so you want to make your model +as small as possible. Furthermore, smaller neural networks can persist in and +out of a mobile device's memory faster. + +To understand how large your network will be on disk, start by looking at the +size on disk of your `GraphDef` file after you’ve run `freeze_graph` and +`strip_unused_nodes` on it (see @{$mobile/prepare_models$Preparing models} for +more details on these tools), since then it should only contain +inference-related nodes. To double-check that your results are as expected, run +the `summarize_graph` tool to see how many parameters are in constants: + + bazel build tensorflow/tools/graph_transforms:summarize_graph && \ + bazel-bin/tensorflow/tools/graph_transforms/summarize_graph \ + --in_graph=/tmp/tensorflow_inception_graph.pb + +That command should give you output that looks something like this: + + No inputs spotted. + Found 1 possible outputs: (name=softmax, op=Softmax) + Found 23885411 (23.89M) const parameters, 0 (0) variable parameters, + and 99 control_edges + Op types used: 489 Const, 99 CheckNumerics, 99 Identity, 94 + BatchNormWithGlobalNormalization, 94 Conv2D, 94 Relu, 11 Concat, 9 AvgPool, + 5 MaxPool, 1 Sub, 1 Softmax, 1 ResizeBilinear, 1 Reshape, 1 Mul, 1 MatMul, + 1 ExpandDims, 1 DecodeJpeg, 1 Cast, 1 BiasAdd + +The important part for our current purposes is the number of const +parameters. In most models these will be stored as 32-bit floats to start, so if +you multiply the number of const parameters by four, you should get something +that’s close to the size of the file on disk. You can often get away with only +eight-bits per parameter with very little loss of accuracy in the final result, +so if your file size is too large you can try using +@{$performance/quantization$quantize_weights} to transform the parameters down. + + bazel build tensorflow/tools/graph_transforms:transform_graph && \ + bazel-bin/tensorflow/tools/graph_transforms/transform_graph \ + --in_graph=/tmp/tensorflow_inception_optimized.pb \ + --out_graph=/tmp/tensorflow_inception_quantized.pb \ + --inputs='Mul:0' --outputs='softmax:0' --transforms='quantize_weights' + +If you look at the resulting file size, you should see that it’s about a quarter +of the original at 23MB. + +Another transform is `round_weights`, which doesn't make the file smaller, but it +makes the file compressable to about the same size as when `quantize_weights` is +used. This is particularly useful for mobile development, taking advantage of +the fact that app bundles are compressed before they’re downloaded by consumers. + +The original file does not compress well with standard algorithms, because the +bit patterns of even very similar numbers can be very different. The +`round_weights` transform keeps the weight parameters stored as floats, but +rounds them to a set number of step values. This means there are a lot more +repeated byte patterns in the stored model, and so compression can often bring +the size down dramatically, in many cases to near the size it would be if they +were stored as eight bit. + +Another advantage of `round_weights` is that the framework doesn’t have to +allocate a temporary buffer to unpack the parameters into, as we have to when +we just use `quantize_weights`. This saves a little bit of latency (though the +results should be cached so it’s only costly on the first run) and makes it +possible to use memory mapping, as described later. + +## Binary Size + +One of the biggest differences between mobile and server development is the +importance of binary size. On desktop machines it’s not unusual to have +executables that are hundreds of megabytes on disk, but for mobile and embedded +apps it’s vital to keep the binary as small as possible so that user downloads +are easy. As mentioned above, TensorFlow only includes a subset of op +implementations by default, but this still results in a 12 MB final +executable. To reduce this, you can set up the library to only include the +implementations of the ops that you actually need, based on automatically +analyzing your model. To use it: + +- Run `tools/print_required_ops/print_selective_registration_header.py` on your + model to produce a header file that only enables the ops it uses. + +- Place the `ops_to_register.h` file somewhere that the compiler can find + it. This can be in the root of your TensorFlow source folder. + +- Build TensorFlow with `SELECTIVE_REGISTRATION` defined, for example by passing + in `--copts=”-DSELECTIVE_REGISTRATION”` to your Bazel build command. + +This process recompiles the library so that only the needed ops and types are +included, which can dramatically reduce the executable size. For example, with +Inception v3, the new size is only 1.5MB. + +## How to Profile your Model + +Once you have an idea of what your device's peak performance range is, it’s +worth looking at its actual current performance. Using a standalone TensorFlow +benchmark, rather than running it inside a larger app, helps isolate just the +Tensorflow contribution to the +latency. The +[tensorflow/tools/benchmark](https://www.tensorflow.org/code/tensorflow/tools/benchmark/) tool +is designed to help you do this. To run it on Inception v3 on your desktop +machine, build this benchmark model: + + bazel build -c opt tensorflow/tools/benchmark:benchmark_model && \ + bazel-bin/tensorflow/tools/benchmark/benchmark_model \ + --graph=/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \ + --input_layer_shape="1,299,299,3" --input_layer_type="float" \ + --output_layer="softmax:0" --show_run_order=false --show_time=false \ + --show_memory=false --show_summary=true --show_flops=true --logtostderr + +You should see output that looks something like this: + +
+============================== Top by Computation Time ==============================
+[node
+ type]  [start]  [first] [avg ms]     [%]  [cdf%]  [mem KB]  [Name]
+Conv2D   22.859   14.212   13.700  4.972%  4.972%  3871.488  conv_4/Conv2D
+Conv2D    8.116    8.964   11.315  4.106%  9.078%  5531.904  conv_2/Conv2D
+Conv2D   62.066   16.504    7.274  2.640% 11.717%   443.904  mixed_3/conv/Conv2D
+Conv2D    2.530    6.226    4.939  1.792% 13.510%  2765.952  conv_1/Conv2D
+Conv2D   55.585    4.605    4.665  1.693% 15.203%   313.600  mixed_2/tower/conv_1/Conv2D
+Conv2D  127.114    5.469    4.630  1.680% 16.883%    81.920  mixed_10/conv/Conv2D
+Conv2D   47.391    6.994    4.588  1.665% 18.548%   313.600  mixed_1/tower/conv_1/Conv2D
+Conv2D   39.463    7.878    4.336  1.574% 20.122%   313.600  mixed/tower/conv_1/Conv2D
+Conv2D  127.113    4.192    3.894  1.413% 21.535%   114.688  mixed_10/tower_1/conv/Conv2D
+Conv2D   70.188    5.205    3.626  1.316% 22.850%   221.952  mixed_4/conv/Conv2D
+
+============================== Summary by node type ==============================
+[Node type]  [count]  [avg ms]    [avg %]    [cdf %]  [mem KB]
+Conv2D            94   244.899    88.952%    88.952% 35869.953
+BiasAdd           95     9.664     3.510%    92.462% 35873.984
+AvgPool            9     7.990     2.902%    95.364%  7493.504
+Relu              94     5.727     2.080%    97.444% 35869.953
+MaxPool            5     3.485     1.266%    98.710%  3358.848
+Const            192     1.727     0.627%    99.337%     0.000
+Concat            11     1.081     0.393%    99.730%  9892.096
+MatMul             1     0.665     0.242%    99.971%     4.032
+Softmax            1     0.040     0.015%    99.986%     4.032
+<>                 1     0.032     0.012%    99.997%     0.000
+Reshape            1     0.007     0.003%   100.000%     0.000
+
+Timings (microseconds): count=50 first=330849 curr=274803 min=232354 max=415352 avg=275563 std=44193
+Memory (bytes): count=50 curr=128366400(all same)
+514 nodes defined 504 nodes observed
+
+ +This is the summary view, which is enabled by the show_summary flag. To +interpret it, the first table is a list of the nodes that took the most time, in +order by how long they took. From left to right, the columns are: + +- Node type, what kind of operation this was. + +- Start time of the op, showing where it falls in the sequence of operations. + +- First time in milliseconds. This is how long the operation took on the first + run of the benchmark, since by default 20 runs are executed to get more + reliable statistics. The first time is useful to spot which ops are doing + expensive calculations on the first run, and then caching the results. + +- Average time for the operation across all runs, in milliseconds. + +- What percentage of the total time for one run the op took. This is useful to + understand where the hotspots are. + +- The cumulative total time of this and the previous ops in the table. This is + handy for understanding what the distribution of work is across the layers, to + see if just a few of the nodes are taking up most of the time. + +- Name of the node. + +The second table is similar, but instead of breaking down the timings by +particular named nodes, it groups them by the kind of op. This is very useful to +understand which op implementations you might want to optimize or eliminate from +your graph. The table is arranged with the most costly operations at the start, +and only shows the top ten entries, with a placeholder for other nodes. The +columns from left to right are: + +- Type of the nodes being analyzed. + +- Accumulated average time taken by all nodes of this type, in milliseconds. + +- What percentage of the total time was taken by this type of operation. + +- Cumulative time taken by this and op types higher in the table, so you can + understand the distribution of the workload. + +- How much memory the outputs of this op type took up. + +Both of these tables are set up so that you can easily copy and paste their +results into spreadsheet documents, since they are output with tabs as +separators between the columns. The summary by node type can be the most useful +when looking for optimization opportunities, since it’s a pointer to the code +that’s taking the most time. In this case, you can see that the Conv2D ops are +almost 90% of the execution time. This is a sign that the graph is pretty +optimal, since convolutions and matrix multiplies are expected to be the bulk of +a neural network’s computing workload. + +As a rule of thumb, it’s more worrying if you see a lot of other operations +taking up more than a small fraction of the time. For neural networks, the ops +that don’t involve large matrix multiplications should usually be dwarfed by the +ones that do, so if you see a lot of time going into those it’s a sign that +either your network is non-optimally constructed, or the code implementing those +ops is not as optimized as it could +be. [Performance bugs](https://github.com/tensorflow/tensorflow/issues) or +patches are always welcome if you do encounter this situation, especially if +they include an attached model exhibiting this behavior and the command line +used to run the benchmark tool on it. + +The run above was on your desktop, but the tool also works on Android, which is +where it’s most useful for mobile development. Here’s an example command line to +run it on a 64-bit ARM device: + + bazel build -c opt --config=android_arm64 \ + tensorflow/tools/benchmark:benchmark_model + adb push bazel-bin/tensorflow/tools/benchmark/benchmark_model /data/local/tmp + adb push /tmp/tensorflow_inception_graph.pb /data/local/tmp/ + adb shell '/data/local/tmp/benchmark_model \ + --graph=/data/local/tmp/tensorflow_inception_graph.pb --input_layer="Mul" \ + --input_layer_shape="1,299,299,3" --input_layer_type="float" \ + --output_layer="softmax:0" --show_run_order=false --show_time=false \ + --show_memory=false --show_summary=true' + +You can interpret the results in exactly the same way as the desktop version +above. If you have any trouble figuring out what the right input and output +names and types are, take a look at the @{$mobile/prepare_models$Preparing +models} page for details about detecting these for your model, and look at the +`summarize_graph` tool which may give you +helpful information. + +There isn’t good support for command line tools on iOS, so instead there’s a +separate example +at +[tensorflow/examples/ios/benchmark](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark) that +packages the same functionality inside a standalone app. This outputs the +statistics to both the screen of the device and the debug log. If you want +on-screen statistics for the Android example apps, you can turn them on by +pressing the volume-up button. + +## Profiling within your own app + +The output you see from the benchmark tool is generated from modules that are +included as part of the standard TensorFlow runtime, which means you have access +to them within your own applications too. You can see an example of how to do +that [here](https://www.tensorflow.org/code/tensorflow/examples/ios/benchmark/BenchmarkViewController.mm?l=139). + +The basic steps are: + +1. Create a StatSummarizer object: + + tensorflow::StatSummarizer stat_summarizer(tensorflow_graph); + +2. Set up the options: + + tensorflow::RunOptions run_options; + run_options.set_trace_level(tensorflow::RunOptions::FULL_TRACE); + tensorflow::RunMetadata run_metadata; + +3. Run the graph: + + run_status = session->Run(run_options, inputs, output_layer_names, {}, + output_layers, &run_metadata); + +4. Calculate the results and print them out: + + assert(run_metadata.has_step_stats()); + const tensorflow::StepStats& step_stats = run_metadata.step_stats(); + stat_summarizer->ProcessStepStats(step_stats); + stat_summarizer->PrintStepStats(); + +## Visualizing Models + +The most effective way to speed up your code is by altering your model so it +does less work. To do that, you need to understand what your model is doing, and +visualizing it is a good first step. To get a high-level overview of your graph, +use [TensorBoard](https://github.com/tensorflow/tensorboard). + +## Threading + +The desktop version of TensorFlow has a sophisticated threading model, and will +try to run multiple operations in parallel if it can. In our terminology this is +called “inter-op parallelism” (though to avoid confusion with “intra-op”, you +could think of it as “between-op” instead), and can be set by specifying +`inter_op_parallelism_threads` in the session options. + +By default, mobile devices run operations serially; that is, +`inter_op_parallelism_threads` is set to 1. Mobile processors usually have few +cores and a small cache, so running multiple operations accessing disjoint parts +of memory usually doesn’t help performance. “Intra-op parallelism” (or +“within-op”) can be very helpful though, especially for computation-bound +operations like convolutions where different threads can feed off the same small +set of memory. + +On mobile, how many threads an op will use is set to the number of cores by +default, or 2 when the number of cores can't be determined. You can override the +default number of threads that ops are using by setting +`intra_op_parallelism_threads` in the session options. It’s a good idea to +reduce the default if your app has its own threads doing heavy processing, so +that they don’t interfere with each other. + +To see more details on session options, look at [ConfigProto](https://www.tensorflow.org/code/tensorflow/core/protobuf/config.proto). + +## Retrain with mobile data + +The biggest cause of accuracy problems when running models on mobile apps is +unrepresentative training data. For example, most of the Imagenet photos are +well-framed so that the object is in the center of the picture, well-lit, and +shot with a normal lens. Photos from mobile devices are often poorly framed, +badly lit, and can have fisheye distortions, especially selfies. + +The solution is to expand your training set with data actually captured from +your application. This step can involve extra work, since you’ll have to label +the examples yourself, but even if you just use it to expand your original +training data, it can help the training set dramatically. Improving the training +set by doing this, and by fixing other quality issues like duplicates or badly +labeled examples is the single best way to improve accuracy. It’s usually a +bigger help than altering your model architecture or using different techniques. + +## Reducing model loading time and/or memory footprint + +Most operating systems allow you to load a file using memory mapping, rather +than going through the usual I/O APIs. Instead of allocating an area of memory +on the heap and then copying bytes from disk into it, you simply tell the +operating system to make the entire contents of a file appear directly in +memory. This has several advantages: + +* Speeds loading +* Reduces paging (increases performance) +* Does not count towards RAM budget for your app + +TensorFlow has support for memory mapping the weights that form the bulk of most +model files. Because of limitations in the `ProtoBuf` serialization format, we +have to make a few changes to our model loading and processing code. The +way memory mapping works is that we have a single file where the first part is a +normal `GraphDef` serialized into the protocol buffer wire format, but then the +weights are appended in a form that can be directly mapped. + +To create this file, run the +`tensorflow/contrib/util:convert_graphdef_memmapped_format` tool. This takes in +a `GraphDef` file that’s been run through `freeze_graph` and converts it to the +format that has the weights appended at the end. Since that file’s no longer a +standard `GraphDef` protobuf, you then need to make some changes to the loading +code. You can see an example of this in +the +[iOS Camera demo app](https://www.tensorflow.org/code/tensorflow/examples/ios/camera/tensorflow_utils.mm?l=147), +in the `LoadMemoryMappedModel()` function. + +The same code (with the Objective C calls for getting the filenames substituted) +can be used on other platforms too. Because we’re using memory mapping, we need +to start by creating a special TensorFlow environment object that’s set up with +the file we’ll be using: + + std::unique_ptr memmapped_env; + memmapped_env->reset( + new tensorflow::MemmappedEnv(tensorflow::Env::Default())); + tensorflow::Status mmap_status = + (memmapped_env->get())->InitializeFromFile(file_path); + +You then need to pass in this environment to subsequent calls, like this one for +loading the graph: + + tensorflow::GraphDef tensorflow_graph; + tensorflow::Status load_graph_status = ReadBinaryProto( + memmapped_env->get(), + tensorflow::MemmappedFileSystem::kMemmappedPackageDefaultGraphDef, + &tensorflow_graph); + +You also need to create the session with a pointer to the environment you’ve +created: + + tensorflow::SessionOptions options; + options.config.mutable_graph_options() + ->mutable_optimizer_options() + ->set_opt_level(::tensorflow::OptimizerOptions::L0); + options.env = memmapped_env->get(); + + tensorflow::Session* session_pointer = nullptr; + tensorflow::Status session_status = + tensorflow::NewSession(options, &session_pointer); + +One thing to notice here is that we’re also disabling automatic optimizations, +since in some cases these will fold constant sub-trees, and so create copies of +tensor values that we don’t want and use up more RAM. + +Once you’ve gone through these steps, you can use the session and graph as +normal, and you should see a reduction in loading time and memory usage. + +## Protecting model files from easy copying + +By default, your models will be stored in the standard serialized protobuf +format on disk. In theory this means that anybody can copy your model, which you +may not want. However, in practice, most models are so application-specific and +obfuscated by optimizations that the risk is similar to that of competitors +disassembling and reusing your code, but if you do want to make it tougher for +casual users to access your files it is possible to take some basic steps. + +Most of our examples use +the +[ReadBinaryProto()](https://www.tensorflow.org/code/tensorflow/core/platform/env.cc?q=core/platform/env.cc&l=409) convenience +call to load a `GraphDef` from disk. This does require an unencrypted protobuf on +disk. Luckily though, the implementation of the call is pretty straightforward +and it should be easy to write an equivalent that can decrypt in memory. Here's +some code that shows how you can read and decrypt a protobuf using your own +decryption routine: + + Status ReadEncryptedProto(Env* env, const string& fname, + ::tensorflow::protobuf::MessageLite* proto) { + string data; + TF_RETURN_IF_ERROR(ReadFileToString(env, fname, &data)); + + DecryptData(&data); // Your own function here. + + if (!proto->ParseFromString(&data)) { + TF_RETURN_IF_ERROR(stream->status()); + return errors::DataLoss("Can't parse ", fname, " as binary proto"); + } + return Status::OK(); + } + +To use this you’d need to define the DecryptData() function yourself. It could +be as simple as something like: + + void DecryptData(string* data) { + for (int i = 0; i < data.size(); ++i) { + data[i] = data[i] ^ 0x23; + } + } + +You may want something more complex, but exactly what you’ll need is outside the +current scope here. diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md new file mode 100644 index 0000000000000000000000000000000000000000..c5a560e074e3fd51708b6867d12426297decf6ae --- /dev/null +++ b/tensorflow/docs_src/mobile/prepare_models.md @@ -0,0 +1,301 @@ +# Preparing models for mobile deployment + +The requirements for storing model information during training are very +different from when you want to release it as part of a mobile app. This section +covers the tools involved in converting from a training model to something +releasable in production. + +## What is up with all the different saved file formats? + +You may find yourself getting very confused by all the different ways that +TensorFlow can save out graphs. To help, here’s a rundown of some of the +different components, and what they are used for. The objects are mostly defined +and serialized as protocol buffers: + +- [NodeDef](https://www.tensorflow.org/code/tensorflow/core/framework/node_def.proto): + Defines a single operation in a model. It has a unique name, a list of the + names of other nodes it pulls inputs from, the operation type it implements + (for example `Add`, or `Mul`), and any attributes that are needed to control + that operation. This is the basic unit of computation for TensorFlow, and all + work is done by iterating through a network of these nodes, applying each one + in turn. One particular operation type that’s worth knowing about is `Const`, + since this holds information about a constant. This may be a single, scalar + number or string, but it can also hold an entire multi-dimensional tensor + array. The values for a `Const` are stored inside the `NodeDef`, and so large + constants can take up a lot of room when serialized. + +- [Checkpoint](https://www.tensorflow.org/code/tensorflow/core/util/tensor_bundle/tensor_bundle.h). Another + way of storing values for a model is by using `Variable` ops. Unlike `Const` + ops, these don’t store their content as part of the `NodeDef`, so they take up + very little space within the `GraphDef` file. Instead their values are held in + RAM while a computation is running, and then saved out to disk as checkpoint + files periodically. This typically happens as a neural network is being + trained and weights are updated, so it’s a time-critical operation, and it may + happen in a distributed fashion across many workers, so the file format has to + be both fast and flexible. They are stored as multiple checkpoint files, + together with metadata files that describe what’s contained within the + checkpoints. When you’re referring to a checkpoint in the API (for example + when passing a filename in as a command line argument), you’ll use the common + prefix for a set of related files. If you had these files: + + /tmp/model/model-chkpt-1000.data-00000-of-00002 + /tmp/model/model-chkpt-1000.data-00001-of-00002 + /tmp/model/model-chkpt-1000.index + /tmp/model/model-chkpt-1000.meta + + You would refer to them as `/tmp/model/chkpt-1000`. + +- [GraphDef](https://www.tensorflow.org/code/tensorflow/core/framework/graph.proto): + Has a list of `NodeDefs`, which together define the computational graph to + execute. During training, some of these nodes will be `Variables`, and so if + you want to have a complete graph you can run, including the weights, you’ll + need to call a restore operation to pull those values from + checkpoints. Because checkpoint loading has to be flexible to deal with all of + the training requirements, this can be tricky to implement on mobile and + embedded devices, especially those with no proper file system available like + iOS. This is where + the + [`freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py) script + comes in handy. As mentioned above, `Const` ops store their values as part of + the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes, + then we only need a single `GraphDef` file to hold the model architecture and + the weights. Freezing the graph handles the process of loading the + checkpoints, and then converts all Consts to Variables. You can then load the + resulting file in a single call, without having to restore variable values + from checkpoints. One thing to watch out for with `GraphDef` files is that + sometimes they’re stored in text format for easy inspection. These versions + usually have a ‘.pbtxt’ filename suffix, whereas the binary files end with + ‘.pb’. + +- [FunctionDefLibrary](https://www.tensorflow.org/code/tensorflow/core/framework/function.proto): + This appears in `GraphDef`, and is effectively a set of sub-graphs, each with + information about their input and output nodes. Each sub-graph can then be + used as an op in the main graph, allowing easy instantiation of different + nodes, in a similar way to how functions encapsulate code in other languages. + +- [MetaGraphDef](https://www.tensorflow.org/code/tensorflow/core/protobuf/meta_graph.proto): + A plain `GraphDef` only has information about the network of computations, but + doesn’t have any extra information about the model or how it can be + used. `MetaGraphDef` contains a `GraphDef` defining the computation part of + the model, but also includes information like ‘signatures’, which are + suggestions about which inputs and outputs you may want to call the model + with, data on how and where any checkpoint files are saved, and convenience + tags for grouping ops together for ease of use. + +- [SavedModel](https://www.tensorflow.org/code/tensorflow/core/protobuf/saved_model.proto): + It’s common to want to have different versions of a graph that rely on a + common set of variable checkpoints. For example, you might need a GPU and a + CPU version of the same graph, but keep the same weights for both. You might + also need some extra files (like label names) as part of your + model. The + [SavedModel](https://www.tensorflow.org/code/tensorflow/python/saved_model/README.md) format + addresses these needs by letting you save multiple versions of the same graph + without duplicating variables, and also storing asset files in the same + bundle. Under the hood, it uses `MetaGraphDef` and checkpoint files, along + with extra metadata files. It’s the format that you’ll want to use if you’re + deploying a web API using TensorFlow Serving, for example. + +## How do you get a model you can use on mobile? + +In most situations, training a model with TensorFlow will give you a folder +containing a `GraphDef` file (usually ending with the `.pb` or `.pbtxt` extension) and +a set of checkpoint files. What you need for mobile or embedded deployment is a +single `GraphDef` file that’s been ‘frozen’, or had its variables converted into +inline constants so everything’s in one file. To handle the conversion, you’ll +need the `freeze_graph.py` script, that’s held in +[`tensorflow/python/tools/freeze_graph.py`](https://www.tensorflow.org/code/tensorflow/python/tools/freeze_graph.py). You’ll run it like this: + + bazel build tensorflow/tools:freeze_graph + bazel-bin/tensorflow/tools/freeze_graph \ + --input_graph=/tmp/model/my_graph.pb \ + --input_checkpoint=/tmp/model/model.ckpt-1000 \ + --output_graph=/tmp/frozen_graph.pb \ + --output_node_names=output_node \ + +The `input_graph` argument should point to the `GraphDef` file that holds your +model architecture. It’s possible that your `GraphDef` has been stored in a text +format on disk, in which case it’s likely to end in `.pbtxt` instead of `.pb`, +and you should add an extra `--input_binary=false` flag to the command. + +The `input_checkpoint` should be the most recent saved checkpoint. As mentioned +in the checkpoint section, you need to give the common prefix to the set of +checkpoints here, rather than a full filename. + +`output_graph` defines where the resulting frozen `GraphDef` will be +saved. Because it’s likely to contain a lot of weight values that take up a +large amount of space in text format, it’s always saved as a binary protobuf. + +`output_node_names` is a list of the names of the nodes that you want to extract +the results of your graph from. This is needed because the freezing process +needs to understand which parts of the graph are actually needed, and which are +artifacts of the training process, like summarization ops. Only ops that +contribute to calculating the given output nodes will be kept. If you know how +your graph is going to be used, these should just be the names of the nodes you +pass into `Session::Run()` as your fetch targets. The easiest way to find the +node names is to inspect the Node objects while building your graph in python. +Inspecting your graph in TensorBoard is another simple way. You can get some +suggestions on likely outputs by running the [`summarize_graph` tool](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms/README.md#inspecting-graphs). + +Because the output format for TensorFlow has changed over time, there are a +variety of other less commonly used flags available too, like `input_saver`, but +hopefully you shouldn’t need these on graphs trained with modern versions of the +framework. + +## Using the Graph Transform Tool + +A lot of the things you need to do to efficiently run a model on device are +available through the [Graph Transform +Tool](https://www.tensorflow.org/code/tensorflow/tools/graph_transforms/README.md). This +command-line tool takes an input `GraphDef` file, applies the set of rewriting +rules you request, and then writes out the result as a `GraphDef`. See the +documentation for more information on how to build and run this tool. + +### Removing training-only nodes + +TensorFlow `GraphDefs` produced by the training code contain all of the +computation that’s needed for back-propagation and updates of weights, as well +as the queuing and decoding of inputs, and the saving out of checkpoints. All of +these nodes are no longer needed during inference, and some of the operations +like checkpoint saving aren’t even supported on mobile platforms. To create a +model file that you can load on devices you need to delete those unneeded +operations by running the `strip_unused_nodes` rule in the Graph Transform Tool. + +The trickiest part of this process is figuring out the names of the nodes you +want to use as inputs and outputs during inference. You'll need these anyway +once you start to run inference, but you also need them here so that the +transform can calculate which nodes are not needed on the inference-only +path. These may not be obvious from the training code. The easiest way to +determine the node name is to explore the graph with TensorBoard. + +Remember that mobile applications typically gather their data from sensors and +have it as arrays in memory, whereas training typically involves loading and +decoding representations of the data stored on disk. In the case of Inception v3 +for example, there’s a `DecodeJpeg` op at the start of the graph that’s designed +to take JPEG-encoded data from a file retrieved from disk and turn it into an +arbitrary-sized image. After that there’s a `BilinearResize` op to scale it to +the expected size, followed by a couple of other ops that convert the byte data +into float and scale the value magnitudes it in the way the rest of the graph +expects. A typical mobile app will skip most of these steps because it’s getting +its input directly from a live camera, so the input node you will actually +supply will be the output of the `Mul` node in this case. + + + +You’ll need to do a similar process of inspection to figure out the correct +output nodes. + +If you’ve just been given a frozen `GraphDef` file, and are not sure about the +contents, try using the `summarize_graph` tool to print out information +about the inputs and outputs it finds from the graph structure. Here’s an +example with the original Inception v3 file: + + bazel run tensorflow/tools/graph_transforms:summarize_graph -- + --in_graph=tensorflow_inception_graph.pb + +Once you have an idea of what the input and output nodes are, you can feed them +into the graph transform tool as the `--input_names` and `--output_names` +arguments, and call the `strip_unused_nodes` transform, like this: + + bazel run tensorflow/tools/graph_transforms:transform_graph -- + --in_graph=tensorflow_inception_graph.pb + --out_graph=optimized_inception_graph.pb --inputs='Mul' --outputs='softmax' + --transforms=' + strip_unused_nodes(type=float, shape="1,299,299,3") + fold_constants(ignore_errors=true) + fold_batch_norms + fold_old_batch_norms' + +One thing to look out for here is that you need to specify the size and type +that you want your inputs to be. This is because any values that you’re going to +be passing in as inputs to inference need to be fed to special `Placeholder` op +nodes, and the transform may need to create them if they don’t already exist. In +the case of Inception v3 for example, a `Placeholder` node replaces the old +`Mul` node that used to output the resized and rescaled image array, since we’re +going to be doing that processing ourselves before we call TensorFlow. It keeps +the original name though, which is why we always feed in inputs to `Mul` when we +run a session with our modified Inception graph. + +After you’ve run this process, you’ll have a graph that only contains the actual +nodes you need to run your prediction process. This is the point where it +becomes useful to run metrics on the graph, so it’s worth running +`summarize_graph` again to understand what’s in your model. + +## What ops should you include on mobile? + +There are hundreds of operations available in TensorFlow, and each one has +multiple implementations for different data types. On mobile platforms, the size +of the executable binary that’s produced after compilation is important, because +app download bundles need to be as small as possible for the best user +experience. If all of the ops and data types are compiled into the TensorFlow +library then the total size of the compiled library can be tens of megabytes, so +by default only a subset of ops and data types are included. + +That means that if you load a model file that’s been trained on a desktop +machine, you may see the error “No OpKernel was registered to support Op” when +you load it on mobile. The first thing to try is to make sure you’ve stripped +out any training-only nodes, since the error will occur at load time even if the +op is never executed. If you’re still hitting the same problem once that’s done, +you’ll need to look at adding the op to your built library. + +The criteria for including ops and types fall into several categories: + +- Are they only useful in back-propagation, for gradients? Since mobile is + focused on inference, we don’t include these. + +- Are they useful mainly for other training needs, such as checkpoint saving? + These we leave out. + +- Do they rely on frameworks that aren’t always available on mobile, such as + libjpeg? To avoid extra dependencies we don’t include ops like `DecodeJpeg`. + +- Are there types that aren’t commonly used? We don’t include boolean variants + of ops for example, since we don’t see much use of them in typical inference + graphs. + +These ops are trimmed by default to optimize for inference on mobile, but it is +possible to alter some build files to change the default. After alternating the +build files, you will need to recompile TensorFlow. See below for more details +on how to do this, and also see @{$mobile/optimizing#binary_size$Optimizing} for +more on reducing your binary size. + +### Locate the implementation + +Operations are broken into two parts. The first is the op definition, which +declares the signature of the operation, which inputs, outputs, and attributes +it has. These take up very little space, and so all are included by default. The +implementations of the op computations are done in kernels, which live in the +`tensorflow/core/kernels` folder. You need to compile the C++ file containing +the kernel implementation of the op you need into the library. To figure out +which file that is, you can search for the operation name in the source +files. + +[Here’s an example search in github](https://github.com/search?utf8=%E2%9C%93&q=repo%3Atensorflow%2Ftensorflow+extension%3Acc+path%3Atensorflow%2Fcore%2Fkernels+REGISTER+Mul&type=Code&ref=searchresults). + +You’ll see that this search is looking for the `Mul` op implementation, and it +finds it in `tensorflow/core/kernels/cwise_op_mul_1.cc`. You need to look for +macros beginning with `REGISTER`, with the op name you care about as one of the +string arguments. + +In this case, the implementations are actually broken up across multiple `.cc` +files, so you’d need to include all of them in your build. If you’re more +comfortable using the command line for code search, here’s a grep command that +also locates the right files if you run it from the root of your TensorFlow +repository: + +`grep 'REGISTER.*"Mul"' tensorflow/core/kernels/*.cc` + +### Add the implementation to the build + +If you’re using Bazel, and building for Android, you’ll want to add the files +you’ve found to +the +[`android_extended_ops_group1`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3565) or +[`android_extended_ops_group2`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3632) targets. You +may also need to include any .cc files they depend on in there. If the build +complains about missing header files, add the .h’s that are needed into +the +[`android_extended_ops`](https://www.tensorflow.org/code/tensorflow/core/kernels/BUILD#L3525) target. + +If you’re using a makefile targetting iOS, Raspberry Pi, etc, go to +[`tensorflow/contrib/makefile/tf_op_files.txt`](https://www.tensorflow.org/code/tensorflow/contrib/makefile/tf_op_files.txt) and +add the right implementation files there. diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md index d3aa901becfb9e87d5cefda611a1e42c54004897..da556bd8483b9bfcd753d6201ed401eaca9933f2 100644 --- a/tensorflow/docs_src/performance/performance_guide.md +++ b/tensorflow/docs_src/performance/performance_guide.md @@ -87,6 +87,40 @@ the Dataset API is still strongly recommended. Try to avoid the following: sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) ``` +#### Fused decode and crop + +If inputs are JPEG images that also require cropping, use fused +@{tf.image.decode_and_crop_jpeg} to speed up preprocessing. +`tf.image.decode_and_crop_jpeg` only decodes the part of +the image within the crop window. This significantly speeds up the process if +the crop window is much smaller than the full image. For imagenet data, this +approach could speed up the input pipeline by up to 30%. + +Example Usage: + +```python +def _image_preprocess_fn(image_buffer): + # image_buffer 1-D string Tensor representing the raw JPEG image buffer. + + # Extract image shape from raw JPEG image buffer. + image_shape = tf.image.extract_jpeg_shape(image_buffer) + + # Get a crop window with distorted bounding box. + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + image_shape, ...) + bbox_begin, bbox_size, distort_bbox = sample_distorted_bounding_box + + # Decode and crop image. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + cropped_image = tf.image.decode_and_crop_jpeg(image, crop_window) +``` + +`tf.image.decode_and_crop_jpeg` is available on all platforms. There is no speed +up on Windows due to the use of `libjpeg` vs. `libjpeg-turbo` on other +platforms. + #### Use large files Reading large numbers of small files significantly impacts I/O performance. diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md index 52258cbae7ae934d3d6bee1601511ec2b969683c..91c0d5b8c60725d5d979c5fcde0d30d0ff098491 100644 --- a/tensorflow/docs_src/performance/xla/operation_semantics.md +++ b/tensorflow/docs_src/performance/xla/operation_semantics.md @@ -328,9 +328,9 @@ placed between each of the entries in that dimension, increasing the size of the array. The holes are filled with a no-op value, which for convolution means zeroes. -Dilation of the rhs is also called atrous convolution. For more details, see the -@{tf.nn.atrous_conv2d}. Dilation of the lhs is -also called deconvolution. +Dilation of the rhs is also called atrous convolution. For more details, see +@{tf.nn.atrous_conv2d}. Dilation of the lhs is also called transposed +convolution. For more details, see @{tf.nn.conv2d_transpose}. The output shape has these dimensions, in this order: diff --git a/tensorflow/docs_src/programmers_guide/datasets.md b/tensorflow/docs_src/programmers_guide/datasets.md index bec1bb1bf04b838e11c0d747abece840177e3140..f458cbcef228b60fcce095a9326b5ea36494cde3 100644 --- a/tensorflow/docs_src/programmers_guide/datasets.md +++ b/tensorflow/docs_src/programmers_guide/datasets.md @@ -1,6 +1,6 @@ # Importing Data -The `Dataset` API enables you to build complex input pipelines from +The @{tf.data.Dataset$`Dataset`} API enables you to build complex input pipelines from simple, reusable pieces. For example, the pipeline for an image model might aggregate data from files in a distributed file system, apply random perturbations to each image, and merge randomly selected images into a batch diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md index 3ede42e8f7c3c0ce44eab63cb1d15712d0aea20b..36a016e880213a5305805247a95a19ad954e2c92 100644 --- a/tensorflow/docs_src/programmers_guide/debugger.md +++ b/tensorflow/docs_src/programmers_guide/debugger.md @@ -141,7 +141,8 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at | **`lt`** | | **List dumped tensors.** | `lt` | | | `-n ` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` | | | `-t ` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` | -| | `s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | +| | `-f ` | List only the tensors that pass a registered tensor filter. | `lt -f has_inf_or_nan` | +| | `-s ` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` | | | `-r` | Sort in reverse order. | `lt -r -s dump_size` | | **`pt`** | | **Print value of a dumped tensor.** | | | | `pt ` | Print tensor value. | `pt hidden/Relu:0` | diff --git a/tensorflow/docs_src/programmers_guide/graphs.md b/tensorflow/docs_src/programmers_guide/graphs.md index 10f53fe8f237104fb190a8d6e586f6b77585b249..c08043835a3c575939d170c52f7f28efb5868c21 100644 --- a/tensorflow/docs_src/programmers_guide/graphs.md +++ b/tensorflow/docs_src/programmers_guide/graphs.md @@ -89,7 +89,7 @@ to all API functions in the same context. For example: * Executing `v = tf.Variable(0)` adds to the graph a @{tf.Operation} that will store a writeable tensor value that persists between @{tf.Session.run} calls. The @{tf.Variable} object wraps this operation, and can be used [like a - tensor](#tensor-like-objects), which will read the current value of the + tensor](#tensor-like_objects), which will read the current value of the stored value. The @{tf.Variable} object also has methods such as @{tf.Variable.assign$`assign`} and @{tf.Variable.assign_add$`assign_add`} that create @{tf.Operation} objects that, when executed, update the stored value. @@ -100,7 +100,7 @@ to all API functions in the same context. For example: when run, will apply those gradients to a set of variables. Most programs rely solely on the default graph. However, -see [Dealing with multiple graphs](#dealing-with-multiple-graphs) for more +see [Dealing with multiple graphs](#programming_with_multiple_graphs) for more advanced use cases. High-level APIs such as the @{tf.estimator.Estimator} API manage the default graph on your behalf, and--for example--may create different graphs for training and evaluation. diff --git a/tensorflow/docs_src/programmers_guide/tensors.md b/tensorflow/docs_src/programmers_guide/tensors.md index cc4181e75e20191f4a5a8a80afec621288388a10..d6f80430cdbb133a486db69bd30a1fae151e3378 100644 --- a/tensorflow/docs_src/programmers_guide/tensors.md +++ b/tensorflow/docs_src/programmers_guide/tensors.md @@ -197,7 +197,7 @@ For example, here is how to make a vector of zeros with the same size as the number of columns in a given matrix: ``` python -zeros = tf.zeros(tf.shape(my_matrix)[1]) +zeros = tf.zeros(my_matrix.shape[1]) ``` ### Changing the shape of a `tf.Tensor` diff --git a/tensorflow/docs_src/tutorials/audio_recognition.md b/tensorflow/docs_src/tutorials/audio_recognition.md index 670e480b12a8a869605130451251183de9c9c62a..336f4d9c18b45cda2441bc7a83e9698bbd618d22 100644 --- a/tensorflow/docs_src/tutorials/audio_recognition.md +++ b/tensorflow/docs_src/tutorials/audio_recognition.md @@ -25,7 +25,7 @@ python tensorflow/examples/speech_commands/train.py ``` The script will start off by downloading the [Speech Commands -dataset](https://download.tensorflow.org/data/speech_commands_v0.01.tar.gz), +dataset](https://storage.cloud.google.com/download.tensorflow.org/data/speech_commands_v0.01.tar.gz), which consists of 65,000 WAVE audio files of people saying thirty different words. This data was collected by Google and released under a CC BY license, and you can help improve it by [contributing five minutes of your own diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md index 8037c92c736ad572a0933bc67dbefbd0226dd136..e808a3677f2a3e89597ef82cc86dd3646775d693 100644 --- a/tensorflow/docs_src/tutorials/layers.md +++ b/tensorflow/docs_src/tutorials/layers.md @@ -192,7 +192,7 @@ def cnn_model_fn(features, labels, mode): The following sections (with headings corresponding to each code block above) dive deeper into the `tf.layers` code used to create each layer, as well as how to calculate loss, configure the training op, and generate predictions. If -you're already experienced with CNNs and @{$estimators$TensorFlow `Estimator`s}, +you're already experienced with CNNs and @{$extend/estimators$TensorFlow `Estimator`s}, and find the above code intuitive, you may want to skim these sections or just skip ahead to ["Training and Evaluating the CNN MNIST Classifier"](#training-and-evaluating-the-cnn-mnist-classifier). @@ -536,8 +536,8 @@ if mode == tf.estimator.ModeKeys.TRAIN: ``` > Note: For a more in-depth look at configuring training ops for Estimator model -> functions, see @{$estimators#defining-the-training-op-for-the-model$"Defining -> the training op for the model"} in the @{$estimators$"Creating Estimations in +> functions, see @{$extend/estimators#defining-the-training-op-for-the-model$"Defining +> the training op for the model"} in the @{$extend/estimators$"Creating Estimations in > tf.estimator"} tutorial. ### Add evaluation metrics @@ -601,7 +601,7 @@ be saved (here, we specify the temp directory `/tmp/mnist_convnet_model`, but feel free to change to another directory of your choice). > Note: For an in-depth walkthrough of the TensorFlow `Estimator` API, see the -> tutorial @{$estimators$"Creating Estimators in tf.estimator."} +> tutorial @{$extend/estimators$"Creating Estimators in tf.estimator."} ### Set Up a Logging Hook {#set_up_a_logging_hook} @@ -720,7 +720,7 @@ Here, we've achieved an accuracy of 97.3% on our test data set. To learn more about TensorFlow Estimators and CNNs in TensorFlow, see the following resources: -* @{$estimators$Creating Estimators in tf.estimator}. An +* @{$extend/estimators$Creating Estimators in tf.estimator}. An introduction to the TensorFlow Estimator API, which walks through configuring an Estimator, writing a model function, calculating loss, and defining a training op. diff --git a/tensorflow/docs_src/tutorials/linear.md b/tensorflow/docs_src/tutorials/linear.md index 4201a8021b13290e46d52df86de09f01181b972d..a6517549c3635fb5dd251f3c3b7b8f876ab4e922 100644 --- a/tensorflow/docs_src/tutorials/linear.md +++ b/tensorflow/docs_src/tutorials/linear.md @@ -16,7 +16,7 @@ give it a try. This overview uses code samples from the tutorial, but the tutorial walks through the code in greater detail. To understand this overview it will help to have some familiarity -with basic machine learning concepts, and also with @{$estimator$tf.estimator}. +with basic machine learning concepts, and also with @{$get_started/estimator$`tf.estimator`}. [TOC] diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md index 6292c1a01e46c3b22da14d29d9a9318ecba47e1a..ba16e12a723938b7d9a18681aeb9a1a361a319b1 100644 --- a/tensorflow/docs_src/tutorials/wide.md +++ b/tensorflow/docs_src/tutorials/wide.md @@ -383,7 +383,7 @@ API: ```python # set num_epochs to None to get infinite stream of data. m.train( - input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True), + input_fn=input_fn(train_file.name, num_epochs=None, shuffle=True), steps=train_steps) ``` @@ -392,7 +392,7 @@ the labels of the holdout data: ```python results = m.evaluate( - input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False), + input_fn=input_fn(test_file.name, num_epochs=1, shuffle=False), steps=None) print("model directory = %s" % model_dir) for key in sorted(results): diff --git a/tensorflow/examples/get_started/regression/test.py b/tensorflow/examples/get_started/regression/test.py index 652b44f543f5a1ee17742859555f9200772aadfa..0b1477ad963d886d92943d4d31c0d63b56bc1677 100644 --- a/tensorflow/examples/get_started/regression/test.py +++ b/tensorflow/examples/get_started/regression/test.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""A simple smoke test that runs these examples for 1 training iteraton.""" +"""A simple smoke test that runs these examples for 1 training iteration.""" from __future__ import absolute_import from __future__ import division diff --git a/tensorflow/examples/learn/iris.py b/tensorflow/examples/learn/iris.py index 33e8d45801409fa112e27f40b1732c43cda72bc2..0a50b3ba87d70a58794bc35009dc76de2cb71d1e 100644 --- a/tensorflow/examples/learn/iris.py +++ b/tensorflow/examples/learn/iris.py @@ -17,47 +17,94 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np -from sklearn import datasets -from sklearn import metrics -from sklearn import model_selection +import os +import urllib import tensorflow as tf +# Data sets +IRIS_TRAINING = 'iris_training.csv' +IRIS_TRAINING_URL = 'http://download.tensorflow.org/data/iris_training.csv' -X_FEATURE = 'x' # Name of the input feature. +IRIS_TEST = 'iris_test.csv' +IRIS_TEST_URL = 'http://download.tensorflow.org/data/iris_test.csv' + +FEATURE_KEYS = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width'] + + +def maybe_download_iris_data(file_name, download_url): + """Downloads the file and returns the number of data.""" + if not os.path.exists(file_name): + raw = urllib.urlopen(download_url).read() + with open(file_name, 'w') as f: + f.write(raw) + + # The first line is a comma-separated string. The first one is the number of + # total data in the file. + with open(file_name, 'r') as f: + first_line = f.readline() + num_elements = first_line.split(',')[0] + return int(num_elements) + + +def input_fn(file_name, num_data, batch_size, is_training): + """Creates an input_fn required by Estimator train/evaluate.""" + # If the data sets aren't stored locally, download them. + + def _parse_csv(rows_string_tensor): + """Takes the string input tensor and returns tuple of (features, labels).""" + # Last dim is the label. + num_features = len(FEATURE_KEYS) + num_columns = num_features + 1 + columns = tf.decode_csv(rows_string_tensor, + record_defaults=[[]] * num_columns) + features = dict(zip(FEATURE_KEYS, columns[:num_features])) + labels = tf.cast(columns[num_features], tf.int32) + return features, labels + + def _input_fn(): + """The input_fn.""" + dataset = tf.data.TextLineDataset([file_name]) + # Skip the first line (which does not have data). + dataset = dataset.skip(1) + dataset = dataset.map(_parse_csv) + + if is_training: + # For this small dataset, which can fit into memory, to achieve true + # randomness, the shuffle buffer size is set as the total number of + # elements in the dataset. + dataset = dataset.shuffle(num_data) + dataset = dataset.repeat() + + dataset = dataset.batch(batch_size) + iterator = dataset.make_one_shot_iterator() + features, labels = iterator.get_next() + return features, labels + + return _input_fn def main(unused_argv): - # Load dataset. - iris = datasets.load_iris() - x_train, x_test, y_train, y_test = model_selection.train_test_split( - iris.data, iris.target, test_size=0.2, random_state=42) + tf.logging.set_verbosity(tf.logging.INFO) + + num_training_data = maybe_download_iris_data( + IRIS_TRAINING, IRIS_TRAINING_URL) + num_test_data = maybe_download_iris_data(IRIS_TEST, IRIS_TEST_URL) # Build 3 layer DNN with 10, 20, 10 units respectively. feature_columns = [ - tf.feature_column.numeric_column( - X_FEATURE, shape=np.array(x_train).shape[1:])] + tf.feature_column.numeric_column(key, shape=1) for key in FEATURE_KEYS] classifier = tf.estimator.DNNClassifier( feature_columns=feature_columns, hidden_units=[10, 20, 10], n_classes=3) # Train. - train_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: x_train}, y=y_train, num_epochs=None, shuffle=True) - classifier.train(input_fn=train_input_fn, steps=200) - - # Predict. - test_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) - predictions = classifier.predict(input_fn=test_input_fn) - y_predicted = np.array(list(p['class_ids'] for p in predictions)) - y_predicted = y_predicted.reshape(np.array(y_test).shape) - - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - - # Score with tensorflow. + train_input_fn = input_fn(IRIS_TRAINING, num_training_data, batch_size=32, + is_training=True) + classifier.train(input_fn=train_input_fn, steps=400) + + # Eval. + test_input_fn = input_fn(IRIS_TEST, num_test_data, batch_size=32, + is_training=False) scores = classifier.evaluate(input_fn=test_input_fn) print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py index 5344526b52b970721fccdc450e902d42573608dc..88425ea0d0bf72fb7e7d9cbab27da023f3ade122 100644 --- a/tensorflow/examples/learn/mnist.py +++ b/tensorflow/examples/learn/mnist.py @@ -97,6 +97,8 @@ def conv_model(features, labels, mode): def main(unused_args): + tf.logging.set_verbosity(tf.logging.INFO) + ### Download and load MNIST dataset. mnist = tf.contrib.learn.datasets.DATASETS['mnist']('/tmp/mnist') train_input_fn = tf.estimator.inputs.numpy_input_fn( @@ -115,6 +117,7 @@ def main(unused_args): feature_columns = [ tf.feature_column.numeric_column( X_FEATURE, shape=mnist.train.images.shape[1:])] + classifier = tf.estimator.LinearClassifier( feature_columns=feature_columns, n_classes=N_DIGITS) classifier.train(input_fn=train_input_fn, steps=200) diff --git a/tensorflow/examples/learn/random_forest_mnist.py b/tensorflow/examples/learn/random_forest_mnist.py index 3c09990ea1eecdf7b5dff95b0fb60197cd0787b7..72c935cdae2196a1309097e4e6f15bd6f22f96a5 100644 --- a/tensorflow/examples/learn/random_forest_mnist.py +++ b/tensorflow/examples/learn/random_forest_mnist.py @@ -1,4 +1,4 @@ - # Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,18 +21,14 @@ import argparse import sys import tempfile -# pylint: disable=g-backslash-continuation -from tensorflow.contrib.learn.python.learn\ - import metric_spec -from tensorflow.contrib.learn.python.learn.estimators\ - import estimator -from tensorflow.contrib.tensor_forest.client\ - import eval_metrics -from tensorflow.contrib.tensor_forest.client\ - import random_forest -from tensorflow.contrib.tensor_forest.python\ - import tensor_forest +import numpy + +from tensorflow.contrib.learn.python.learn import metric_spec +from tensorflow.contrib.tensor_forest.client import eval_metrics +from tensorflow.contrib.tensor_forest.client import random_forest +from tensorflow.contrib.tensor_forest.python import tensor_forest from tensorflow.examples.tutorials.mnist import input_data +from tensorflow.python.estimator.inputs import numpy_io from tensorflow.python.platform import app FLAGS = None @@ -41,16 +37,15 @@ FLAGS = None def build_estimator(model_dir): """Build an estimator.""" params = tensor_forest.ForestHParams( - num_classes=10, num_features=784, - num_trees=FLAGS.num_trees, max_nodes=FLAGS.max_nodes) + num_classes=10, + num_features=784, + num_trees=FLAGS.num_trees, + max_nodes=FLAGS.max_nodes) graph_builder_class = tensor_forest.RandomForestGraphs if FLAGS.use_training_loss: graph_builder_class = tensor_forest.TrainingLossForest - # Use the SKCompat wrapper, which gives us a convenient way to split - # in-memory data like MNIST into batches. - return estimator.SKCompat(random_forest.TensorForestEstimator( - params, graph_builder_class=graph_builder_class, - model_dir=model_dir)) + return random_forest.TensorForestEstimator( + params, graph_builder_class=graph_builder_class, model_dir=model_dir) def train_and_eval(): @@ -62,18 +57,30 @@ def train_and_eval(): mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=False) - est.fit(x=mnist.train.images, y=mnist.train.labels, - batch_size=FLAGS.batch_size) + train_input_fn = numpy_io.numpy_input_fn( + x={'images': mnist.train.images}, + y=mnist.train.labels.astype(numpy.int32), + batch_size=FLAGS.batch_size, + num_epochs=None, + shuffle=True) + est.fit(input_fn=train_input_fn, steps=None) metric_name = 'accuracy' - metric = {metric_name: - metric_spec.MetricSpec( - eval_metrics.get_metric(metric_name), - prediction_key=eval_metrics.get_prediction_key(metric_name))} - - results = est.score(x=mnist.test.images, y=mnist.test.labels, - batch_size=FLAGS.batch_size, - metrics=metric) + metric = { + metric_name: + metric_spec.MetricSpec( + eval_metrics.get_metric(metric_name), + prediction_key=eval_metrics.get_prediction_key(metric_name)) + } + + test_input_fn = numpy_io.numpy_input_fn( + x={'images': mnist.test.images}, + y=mnist.test.labels.astype(numpy.int32), + num_epochs=1, + batch_size=FLAGS.batch_size, + shuffle=False) + + results = est.evaluate(input_fn=test_input_fn, metrics=metric) for key in sorted(results): print('%s: %s' % (key, results[key])) diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py index 33a09bb6e0a00a18b91242fdafc05d60e382c0ba..1e0966475b01d067330dc4797032d561857fd208 100755 --- a/tensorflow/examples/learn/resnet.py +++ b/tensorflow/examples/learn/resnet.py @@ -190,8 +190,8 @@ def main(unused_args): # Calculate accuracy. test_input_fn = tf.estimator.inputs.numpy_input_fn( - x={X_FEATURE: mnist.train.images}, - y=mnist.train.labels.astype(np.int32), + x={X_FEATURE: mnist.test.images}, + y=mnist.test.labels.astype(np.int32), num_epochs=1, shuffle=False) scores = classifier.evaluate(input_fn=test_input_fn) diff --git a/tensorflow/examples/learn/text_classification.py b/tensorflow/examples/learn/text_classification.py index 26e6e086b302dec48d44e33ac8eb178f06c62a21..ba89c532be5fa0e13a2dcb1f7894be4c631507d7 100644 --- a/tensorflow/examples/learn/text_classification.py +++ b/tensorflow/examples/learn/text_classification.py @@ -91,11 +91,11 @@ def rnn_model(features, labels, mode): word_list = tf.unstack(word_vectors, axis=1) # Create a Gated Recurrent Unit cell with hidden size of EMBEDDING_SIZE. - cell = tf.contrib.rnn.GRUCell(EMBEDDING_SIZE) + cell = tf.nn.rnn_cell.GRUCell(EMBEDDING_SIZE) # Create an unrolled Recurrent Neural Networks to length of # MAX_DOCUMENT_LENGTH and passes word_list as inputs for each unit. - _, encoding = tf.contrib.rnn.static_rnn(cell, word_list, dtype=tf.float32) + _, encoding = tf.nn.static_rnn(cell, word_list, dtype=tf.float32) # Given encoding of RNN, take encoding of last step (e.g hidden size of the # neural network of last step) and pass it as features for softmax @@ -107,6 +107,8 @@ def rnn_model(features, labels, mode): def main(unused_argv): global n_words + tf.logging.set_verbosity(tf.logging.INFO) + # Prepare training and testing data dbpedia = tf.contrib.learn.datasets.load_dataset( 'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data) diff --git a/tensorflow/examples/learn/text_classification_character_cnn.py b/tensorflow/examples/learn/text_classification_character_cnn.py index 5f7c8e73710ff6b9a107e2197fbecc602c074731..363ff003628e03be40c1be6b7b32e12a07533047 100644 --- a/tensorflow/examples/learn/text_classification_character_cnn.py +++ b/tensorflow/examples/learn/text_classification_character_cnn.py @@ -30,7 +30,6 @@ import sys import numpy as np import pandas -from sklearn import metrics import tensorflow as tf FLAGS = None @@ -106,6 +105,8 @@ def char_cnn_model(features, labels, mode): def main(unused_argv): + tf.logging.set_verbosity(tf.logging.INFO) + # Prepare training and testing data dbpedia = tf.contrib.learn.datasets.load_dataset( 'dbpedia', test_with_fake_data=FLAGS.test_with_fake_data, size='large') @@ -130,7 +131,7 @@ def main(unused_argv): train_input_fn = tf.estimator.inputs.numpy_input_fn( x={CHARS_FEATURE: x_train}, y=y_train, - batch_size=len(x_train), + batch_size=128, num_epochs=None, shuffle=True) classifier.train(input_fn=train_input_fn, steps=100) @@ -145,13 +146,9 @@ def main(unused_argv): y_predicted = np.array(list(p['class'] for p in predictions)) y_predicted = y_predicted.reshape(np.array(y_test).shape) - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - # Score with tensorflow. scores = classifier.evaluate(input_fn=test_input_fn) - print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) + print('Accuracy: {0:f}'.format(scores['accuracy'])) if __name__ == '__main__': diff --git a/tensorflow/examples/learn/text_classification_character_rnn.py b/tensorflow/examples/learn/text_classification_character_rnn.py index 1fc9388a1a026013ad14f8d1deeccbed817d1c88..86adc056add508c309b3a5b93e58e9c195995642 100644 --- a/tensorflow/examples/learn/text_classification_character_rnn.py +++ b/tensorflow/examples/learn/text_classification_character_rnn.py @@ -30,7 +30,6 @@ import sys import numpy as np import pandas -from sklearn import metrics import tensorflow as tf FLAGS = None @@ -46,8 +45,8 @@ def char_rnn_model(features, labels, mode): byte_vectors = tf.one_hot(features[CHARS_FEATURE], 256, 1., 0.) byte_list = tf.unstack(byte_vectors, axis=1) - cell = tf.contrib.rnn.GRUCell(HIDDEN_SIZE) - _, encoding = tf.contrib.rnn.static_rnn(cell, byte_list, dtype=tf.float32) + cell = tf.nn.rnn_cell.GRUCell(HIDDEN_SIZE) + _, encoding = tf.nn.static_rnn(cell, byte_list, dtype=tf.float32) logits = tf.layers.dense(encoding, MAX_LABEL, activation=None) @@ -98,28 +97,20 @@ def main(unused_argv): train_input_fn = tf.estimator.inputs.numpy_input_fn( x={CHARS_FEATURE: x_train}, y=y_train, - batch_size=len(x_train), + batch_size=128, num_epochs=None, shuffle=True) classifier.train(input_fn=train_input_fn, steps=100) - # Predict. + # Eval. test_input_fn = tf.estimator.inputs.numpy_input_fn( x={CHARS_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) - predictions = classifier.predict(input_fn=test_input_fn) - y_predicted = np.array(list(p['class'] for p in predictions)) - y_predicted = y_predicted.reshape(np.array(y_test).shape) - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - - # Score with tensorflow. scores = classifier.evaluate(input_fn=test_input_fn) - print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) + print('Accuracy: {0:f}'.format(scores['accuracy'])) if __name__ == '__main__': diff --git a/tensorflow/examples/learn/text_classification_cnn.py b/tensorflow/examples/learn/text_classification_cnn.py index 0ee2405c8bdc35831f29a195791b743161bec80b..be262285a3a7aa0d6b9430a2226b448fe674cd7f 100644 --- a/tensorflow/examples/learn/text_classification_cnn.py +++ b/tensorflow/examples/learn/text_classification_cnn.py @@ -22,7 +22,6 @@ import sys import numpy as np import pandas -from sklearn import metrics import tensorflow as tf FLAGS = None @@ -134,23 +133,15 @@ def main(unused_argv): shuffle=True) classifier.train(input_fn=train_input_fn, steps=100) - # Predict. + # Evaluate. test_input_fn = tf.estimator.inputs.numpy_input_fn( x={WORDS_FEATURE: x_test}, y=y_test, num_epochs=1, shuffle=False) - predictions = classifier.predict(input_fn=test_input_fn) - y_predicted = np.array(list(p['class'] for p in predictions)) - y_predicted = y_predicted.reshape(np.array(y_test).shape) - # Score with sklearn. - score = metrics.accuracy_score(y_test, y_predicted) - print('Accuracy (sklearn): {0:f}'.format(score)) - - # Score with tensorflow. scores = classifier.evaluate(input_fn=test_input_fn) - print('Accuracy (tensorflow): {0:f}'.format(scores['accuracy'])) + print('Accuracy: {0:f}'.format(scores['accuracy'])) if __name__ == '__main__': diff --git a/tensorflow/examples/learn/wide_n_deep_tutorial.py b/tensorflow/examples/learn/wide_n_deep_tutorial.py index 7b9381311c22a1bce179e309b36997ccc3afaac3..e447b3e24e75f0596423babfe438dc908393b7cc 100644 --- a/tensorflow/examples/learn/wide_n_deep_tutorial.py +++ b/tensorflow/examples/learn/wide_n_deep_tutorial.py @@ -107,6 +107,9 @@ deep_columns = [ ] +FLAGS = None + + def maybe_download(train_data, test_data): """Maybe downloads training data and returns train and test file names.""" if train_data: @@ -154,7 +157,14 @@ def build_estimator(model_dir, model_type): def input_fn(data_file, num_epochs, shuffle): - """Input builder function.""" + """Returns an `input_fn` required by Estimator train/evaluate. + + Args: + data_file: The file path to the dataset. + num_epochs: Number of epochs to iterate over data. If `None`, `input_fn` + will generate infinite stream of data. + shuffle: bool, whether to read the data in random order. + """ df_data = pd.read_csv( tf.gfile.Open(data_file), names=CSV_COLUMNS, @@ -164,43 +174,42 @@ def input_fn(data_file, num_epochs, shuffle): # remove NaN elements df_data = df_data.dropna(how="any", axis=0) labels = df_data["income_bracket"].apply(lambda x: ">50K" in x).astype(int) + return tf.estimator.inputs.pandas_input_fn( x=df_data, y=labels, batch_size=100, num_epochs=num_epochs, shuffle=shuffle, - num_threads=5) + num_threads=1) -def train_and_eval(model_dir, model_type, train_steps, train_data, test_data): - """Train and evaluate the model.""" - train_file_name, test_file_name = maybe_download(train_data, test_data) +def main(_): + tf.logging.set_verbosity(tf.logging.INFO) + + train_file_name, test_file_name = maybe_download(FLAGS.train_data, + FLAGS.test_data) + # Specify file path below if want to find the output easily - model_dir = tempfile.mkdtemp() if not model_dir else model_dir + model_dir = FLAGS.model_dir if FLAGS.model_dir else tempfile.mkdtemp() - m = build_estimator(model_dir, model_type) - # set num_epochs to None to get infinite stream of data. - m.train( + estimator = build_estimator(model_dir, FLAGS.model_type) + + # `tf.estimator.TrainSpec`, `tf.estimator.EvalSpec`, and + # `tf.estimator.train_and_evaluate` API are available in TF 1.4. + train_spec = tf.estimator.TrainSpec( input_fn=input_fn(train_file_name, num_epochs=None, shuffle=True), - steps=train_steps) - # set steps to None to run evaluation until all data consumed. - results = m.evaluate( + max_steps=FLAGS.train_steps) + + eval_spec = tf.estimator.EvalSpec( input_fn=input_fn(test_file_name, num_epochs=1, shuffle=False), + # set steps to None to run evaluation until all data consumed. steps=None) - print("model directory = %s" % model_dir) - for key in sorted(results): - print("%s: %s" % (key, results[key])) - # Manual cleanup - shutil.rmtree(model_dir) - - -FLAGS = None + tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec) -def main(_): - train_and_eval(FLAGS.model_dir, FLAGS.model_type, FLAGS.train_steps, - FLAGS.train_data, FLAGS.test_data) + # Manual cleanup + shutil.rmtree(model_dir) if __name__ == "__main__": diff --git a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py index af89c8c77bc01891dfe683904873c96b0aa0fff8..35ca1b2f7f3eccae5ce408fecf5462939edf5507 100644 --- a/tensorflow/examples/tutorials/mnist/fully_connected_feed.py +++ b/tensorflow/examples/tutorials/mnist/fully_connected_feed.py @@ -109,7 +109,7 @@ def do_eval(sess, labels_placeholder) true_count += sess.run(eval_correct, feed_dict=feed_dict) precision = float(true_count) / num_examples - print(' Num examples: %d Num correct: %d Precision @ 1: %0.04f' % + print('Num examples: %d Num correct: %d Precision @ 1: %0.04f' % (num_examples, true_count, precision)) diff --git a/tensorflow/examples/tutorials/mnist/mnist_deep.py b/tensorflow/examples/tutorials/mnist/mnist_deep.py index 4b5b50400a717ec2f29bab8a5fe3171b0e993476..a4dbab5123d49ee97445a5921a14bd1764593025 100644 --- a/tensorflow/examples/tutorials/mnist/mnist_deep.py +++ b/tensorflow/examples/tutorials/mnist/mnist_deep.py @@ -82,7 +82,7 @@ def deepnn(x): W_fc1 = weight_variable([7 * 7 * 64, 1024]) b_fc1 = bias_variable([1024]) - h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64]) + h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) # Dropout - controls the complexity of the model, prevents co-adaptation of diff --git a/tensorflow/examples/udacity/Dockerfile b/tensorflow/examples/udacity/Dockerfile index 3d48ced41b2fc284ddb9454412fdaa3bb2807e6c..3ca58566c1ddb4c2446f7d9b19ee31fb8b603909 100644 --- a/tensorflow/examples/udacity/Dockerfile +++ b/tensorflow/examples/udacity/Dockerfile @@ -1,5 +1,5 @@ FROM gcr.io/tensorflow/tensorflow:latest -MAINTAINER Vincent Vanhoucke +LABEL maintainer="Vincent Vanhoucke " # Pillow needs libjpeg by default as of 3.0. RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 8f5ee9c3df01cb53c84460d12c190ff58bbca5bc..f3160969630db48b0c6562f1d143c188c1116564 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -1849,7 +1849,7 @@ func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf // ``` // // Arguments: -// input: Rank k tensor where k is 2, 4, or 6. +// input: Rank k tensor where k is even and not zero. // // Returns The extracted diagonal. func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) { @@ -1904,10 +1904,10 @@ func DequantizeMode(value string) DequantizeAttr { // If the mode is 'MIN_FIRST', then this approach is used: // // ```c++ -// number_of_steps = 1 << (# of bits in T) -// range_adjust = number_of_steps / (number_of_steps - 1) +// num_discrete_values = 1 << (# of bits in T) +// range_adjust = num_discrete_values / (num_discrete_values - 1) // range = (range_max - range_min) * range_adjust -// range_scale = range / number_of_steps +// range_scale = range / num_discrete_values // const double offset_input = static_cast(input) - lowest_quantized; // result = range_min + ((input - numeric_limits::min()) * range_scale) // ``` @@ -2324,6 +2324,45 @@ func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (aud return op.Output(0), op.Output(1) } +// Elementwise computes the bitwise right-shift of `x` and `y`. +// +// Performs a logical shift for unsigned integer types, and an arithmetic shift +// for signed integer types. +// +// If `y` is negative, or greater than or equal to than the width of `x` in bits +// the result is implementation defined. +func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "RightShift", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Elementwise computes the bitwise left-shift of `x` and `y`. +// +// If `y` is negative, or greater than or equal to the width of `x` in bits the +// result is implementation defined. +func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "LeftShift", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Elementwise computes the bitwise AND of `x` and `y`. // // The result will have those bits set, that are set in both `x` and `y`. The @@ -4101,7 +4140,7 @@ func TensorArraySplitV3(scope *Scope, handle tf.Output, value tf.Output, lengths // ``` // // Arguments: -// diagonal: Rank k tensor where k is at most 3. +// diagonal: Rank k tensor where k is at most 1. func Diag(scope *Scope, diagonal tf.Output) (output tf.Output) { if scope.Err() != nil { return @@ -4893,80 +4932,6 @@ func PriorityQueueV2(scope *Scope, shapes []tf.Shape, optional ...PriorityQueueV return op.Output(0) } -// FIFOQueueV2Attr is an optional argument to FIFOQueueV2. -type FIFOQueueV2Attr func(optionalAttr) - -// FIFOQueueV2Shapes sets the optional shapes attribute to value. -// -// value: The shape of each component in a value. The length of this attr must -// be either 0 or the same as the length of component_types. If the length of -// this attr is 0, the shapes of queue elements are not constrained, and -// only one element may be dequeued at a time. -// If not specified, defaults to <> -// -// REQUIRES: len(value) >= 0 -func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["shapes"] = value - } -} - -// FIFOQueueV2Capacity sets the optional capacity attribute to value. -// -// value: The upper bound on the number of elements in this queue. -// Negative numbers mean no limit. -// If not specified, defaults to -1 -func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["capacity"] = value - } -} - -// FIFOQueueV2Container sets the optional container attribute to value. -// -// value: If non-empty, this queue is placed in the given container. -// Otherwise, a default container is used. -// If not specified, defaults to "" -func FIFOQueueV2Container(value string) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// FIFOQueueV2SharedName sets the optional shared_name attribute to value. -// -// value: If non-empty, this queue will be shared under the given name -// across multiple sessions. -// If not specified, defaults to "" -func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// A queue that produces elements in first-in first-out order. -// -// Arguments: -// component_types: The type of each component in a value. -// -// Returns The handle to the queue. -func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"component_types": component_types} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "FIFOQueueV2", - - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // StridedSliceAttr is an optional argument to StridedSlice. type StridedSliceAttr func(optionalAttr) @@ -5346,6 +5311,101 @@ func DynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged return op.Output(0) } +// FIFOQueueV2Attr is an optional argument to FIFOQueueV2. +type FIFOQueueV2Attr func(optionalAttr) + +// FIFOQueueV2Shapes sets the optional shapes attribute to value. +// +// value: The shape of each component in a value. The length of this attr must +// be either 0 or the same as the length of component_types. If the length of +// this attr is 0, the shapes of queue elements are not constrained, and +// only one element may be dequeued at a time. +// If not specified, defaults to <> +// +// REQUIRES: len(value) >= 0 +func FIFOQueueV2Shapes(value []tf.Shape) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["shapes"] = value + } +} + +// FIFOQueueV2Capacity sets the optional capacity attribute to value. +// +// value: The upper bound on the number of elements in this queue. +// Negative numbers mean no limit. +// If not specified, defaults to -1 +func FIFOQueueV2Capacity(value int64) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["capacity"] = value + } +} + +// FIFOQueueV2Container sets the optional container attribute to value. +// +// value: If non-empty, this queue is placed in the given container. +// Otherwise, a default container is used. +// If not specified, defaults to "" +func FIFOQueueV2Container(value string) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// FIFOQueueV2SharedName sets the optional shared_name attribute to value. +// +// value: If non-empty, this queue will be shared under the given name +// across multiple sessions. +// If not specified, defaults to "" +func FIFOQueueV2SharedName(value string) FIFOQueueV2Attr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// A queue that produces elements in first-in first-out order. +// +// Arguments: +// component_types: The type of each component in a value. +// +// Returns The handle to the queue. +func FIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...FIFOQueueV2Attr) (handle tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"component_types": component_types} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "FIFOQueueV2", + + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Converts the given `resource_handle` representing an iterator to a variant tensor. +// +// Arguments: +// resource_handle: A handle to an iterator resource. +// +// Returns A variant tensor storing the state of the iterator contained in the +// resource. +func SerializeIterator(scope *Scope, resource_handle tf.Output) (serialized tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SerializeIterator", + Input: []tf.Input{ + resource_handle, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Return a tensor with the same shape and contents as the input tensor or value. func Identity(scope *Scope, input tf.Output) (output tf.Output) { if scope.Err() != nil { @@ -5536,40 +5596,6 @@ func IteratorGetNext(scope *Scope, iterator tf.Output, output_types []tf.DataTyp return components } -// Restores the state of the `iterator` from the checkpoint saved at `path` using "SaveIterator". -// -// Returns the created operation. -func RestoreIterator(scope *Scope, iterator tf.Output, path tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "RestoreIterator", - Input: []tf.Input{ - iterator, path, - }, - } - return scope.AddOperation(opspec) -} - -// Saves the state of the `iterator` at `path`. -// -// This state can be restored using "RestoreIterator". -// -// Returns the created operation. -func SaveIterator(scope *Scope, iterator tf.Output, path tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SaveIterator", - Input: []tf.Input{ - iterator, path, - }, - } - return scope.AddOperation(opspec) -} - // Makes a new iterator from the given `dataset` and stores it in `iterator`. // // This operation may be executed multiple times. Each execution will reset the @@ -5880,6 +5906,27 @@ func TensorArrayConcatV2(scope *Scope, handle tf.Output, flow_in tf.Output, dtyp return op.Output(0), op.Output(1) } +// Converts the given variant tensor to an iterator and stores it in the given resource. +// +// Arguments: +// resource_handle: A handle to an iterator resource. +// serialized: A variant tensor storing the state of the iterator contained in the +// resource. +// +// Returns the created operation. +func DeserializeIterator(scope *Scope, resource_handle tf.Output, serialized tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "DeserializeIterator", + Input: []tf.Input{ + resource_handle, serialized, + }, + } + return scope.AddOperation(opspec) +} + // Concatenates tensors along one dimension. // // Arguments: @@ -7786,63 +7833,66 @@ func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer return op.Output(0) } -// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad. -type FusedBatchNormGradAttr func(optionalAttr) - -// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value. +// Adjust the saturation of one or more images. // -// value: A small float number added to the variance of x. -// If not specified, defaults to 0.0001 -func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr { - return func(m optionalAttr) { - m["epsilon"] = value - } -} - -// FusedBatchNormGradDataFormat sets the optional data_format attribute to value. +// `images` is a tensor of at least 3 dimensions. The last dimension is +// interpretted as channels, and must be three. // -// value: The data format for y_backprop, x, x_backprop. -// Either "NHWC" (default) or "NCHW". -// If not specified, defaults to "NHWC" -func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr { - return func(m optionalAttr) { - m["data_format"] = value - } -} - -// FusedBatchNormGradIsTraining sets the optional is_training attribute to value. +// The input image is considered in the RGB colorspace. Conceptually, the RGB +// colors are first mapped into HSV. A scale is then applied all the saturation +// values, and then remapped back to RGB colorspace. // -// value: A bool value to indicate the operation is for training (default) -// or inference. +// Arguments: +// images: Images to adjust. At least 3-D. +// scale: A float scale to add to the saturation. +// +// Returns The hue-adjusted image or images. +func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "AdjustSaturation", + Input: []tf.Input{ + images, scale, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2. +type SelfAdjointEigV2Attr func(optionalAttr) + +// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value. +// +// value: If `True` then eigenvectors will be computed and returned in `v`. +// Otherwise, only the eigenvalues will be computed. // If not specified, defaults to true -func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr { +func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr { return func(m optionalAttr) { - m["is_training"] = value + m["compute_v"] = value } } -// Gradient for batch normalization. +// Computes the eigen decomposition of one or more square self-adjoint matrices. // -// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW". -// The size of 1D Tensors matches the dimension C of the 4D Tensors. +// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in +// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. +// +// ```python +// # a is a tensor. +// # e is a tensor of eigenvalues. +// # v is a tensor of eigenvectors. +// e, v = self_adjoint_eig(a) +// e = self_adjoint_eig(a, compute_v=False) +// ``` // // Arguments: -// y_backprop: A 4D Tensor for the gradient with respect to y. -// x: A 4D Tensor for input data. -// scale: A 1D Tensor for scaling factor, to scale the normalized x. -// reserve_space_1: When is_training is True, a 1D Tensor for the computed batch -// mean to be reused in gradient computation. When is_training is -// False, a 1D Tensor for the population mean to be reused in both -// 1st and 2nd order gradient computation. -// reserve_space_2: When is_training is True, a 1D Tensor for the computed batch -// variance (inverted variance in the cuDNN case) to be reused in -// gradient computation. When is_training is False, a 1D Tensor -// for the population variance to be reused in both 1st and 2nd -// order gradient computation. +// input: `Tensor` input of shape `[N, N]`. // -// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input -// in FusedBatchNorm. -func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradAttr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) { +// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`. +func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) { if scope.Err() != nil { return } @@ -7851,79 +7901,99 @@ func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale t a(attrs) } opspec := tf.OpSpec{ - Type: "FusedBatchNormGrad", + Type: "SelfAdjointEigV2", Input: []tf.Input{ - y_backprop, x, scale, reserve_space_1, reserve_space_2, + input, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4) -} - -// ShapeAttr is an optional argument to Shape. -type ShapeAttr func(optionalAttr) - -// ShapeOutType sets the optional out_type attribute to value. -// If not specified, defaults to DT_INT32 -func ShapeOutType(value tf.DataType) ShapeAttr { - return func(m optionalAttr) { - m["out_type"] = value - } + return op.Output(0), op.Output(1) } -// Returns the shape of a tensor. +// Computes the Eigen Decomposition of a batch of square self-adjoint matrices. // -// This operation returns a 1-D integer tensor representing the shape of `input`. +// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead. // -// For example: +// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices, with the same constraints as the single matrix +// SelfAdjointEig. // -// ``` -// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]] -// shape(t) ==> [2, 2, 3] -// ``` -func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) { +// The result is a [..., M+1, M] matrix with [..., 0,:] containing the +// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. +// +// Arguments: +// input: Shape is `[..., M, M]`. +// +// Returns Shape is `[..., M+1, M]`. +func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } opspec := tf.OpSpec{ - Type: "Shape", + Type: "SelfAdjointEig", Input: []tf.Input{ input, }, - Attrs: attrs, } op := scope.AddOperation(opspec) return op.Output(0) } -// Computes softmax cross entropy cost and gradients to backpropagate. +// Writes contents to the file at input filename. Creates file and recursively // -// Inputs are the logits, not probabilities. +// creates directory if not existing. // // Arguments: -// features: batch_size x num_classes matrix -// labels: batch_size x num_classes matrix -// The caller must ensure that each batch of labels represents a valid -// probability distribution. +// filename: scalar. The name of the file to which we write the contents. +// contents: scalar. The content to be written to the output file. // -// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix). -func SoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) { +// Returns the created operation. +func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "SoftmaxCrossEntropyWithLogits", + Type: "WriteFile", Input: []tf.Input{ - features, labels, + filename, contents, + }, + } + return scope.AddOperation(opspec) +} + +// Computes the Cholesky decomposition of one or more square matrices. +// +// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices. +// +// The input has to be symmetric and positive definite. Only the lower-triangular +// part of the input will be used for this operation. The upper-triangular part +// will not be read. +// +// The output is a tensor of the same shape as the input +// containing the Cholesky decompositions for all input submatrices `[..., :, :]`. +// +// **Note**: The gradient computation on GPU is faster for large matrices but +// not for large batch dimensions when the submatrices are small. In this +// case it might be faster to use the CPU. +// +// Arguments: +// input: Shape is `[..., M, M]`. +// +// Returns Shape is `[..., M, M]`. +func Cholesky(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Cholesky", + Input: []tf.Input{ + input, }, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) + return op.Output(0) } // MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad. @@ -10102,7 +10172,7 @@ func DecodeJSONExample(scope *Scope, json_examples tf.Output) (binary_examples t // Requires `updates.shape = indices.shape + ref.shape[1:]`. // //
-// +// //
// // Arguments: @@ -10520,6 +10590,36 @@ func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...Ran return op.Output(0) } +// Returns the element-wise sum of a list of tensors. +// +// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not +// wait for all of its inputs to be ready before beginning to sum. This can +// save memory if inputs are ready at different times, since minimum temporary +// storage is proportional to the output size rather than the inputs size. +// +// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable. +// +// Returns a `Tensor` of same shape and type as the elements of `inputs`. +// +// Arguments: +// inputs: A list of `Tensor` objects, each with same shape and type. +// shape: Shape of elements of `inputs`. +func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"shape": shape} + opspec := tf.OpSpec{ + Type: "AccumulateNV2", + Input: []tf.Input{ + tf.OutputList(inputs), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Computes the gradient for the inverse of `x` wrt its input. // // Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy` @@ -12483,62 +12583,6 @@ func AvgPool(scope *Scope, value tf.Output, ksize []int64, strides []int64, padd return op.Output(0) } -// Writes contents to the file at input filename. Creates file and recursively -// -// creates directory if not existing. -// -// Arguments: -// filename: scalar. The name of the file to which we write the contents. -// contents: scalar. The content to be written to the output file. -// -// Returns the created operation. -func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "WriteFile", - Input: []tf.Input{ - filename, contents, - }, - } - return scope.AddOperation(opspec) -} - -// Computes the Cholesky decomposition of one or more square matrices. -// -// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices. -// -// The input has to be symmetric and positive definite. Only the lower-triangular -// part of the input will be used for this operation. The upper-triangular part -// will not be read. -// -// The output is a tensor of the same shape as the input -// containing the Cholesky decompositions for all input submatrices `[..., :, :]`. -// -// **Note**: The gradient computation on GPU is faster for large matrices but -// not for large batch dimensions when the submatrices are small. In this -// case it might be faster to use the CPU. -// -// Arguments: -// input: Shape is `[..., M, M]`. -// -// Returns Shape is `[..., M, M]`. -func Cholesky(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Cholesky", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Slice a `SparseTensor` based on the `start` and `size`. // // For example, if the input is @@ -13282,71 +13326,10 @@ func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) { return op.Output(0) } -// Encode audio data using the WAV file format. -// -// This operation will generate a string suitable to be saved out to create a .wav -// audio file. It will be encoded in the 16-bit PCM format. It takes in float -// values in the range -1.0f to 1.0f, and any outside that value will be clamped to -// that range. -// -// `audio` is a 2-D float Tensor of shape `[length, channels]`. -// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100). +// Read an element from the TensorArray into output `value`. // // Arguments: -// audio: 2-D with shape `[length, channels]`. -// sample_rate: Scalar containing the sample frequency. -// -// Returns 0-D. WAV-encoded file contents. -func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "EncodeWav", - Input: []tf.Input{ - audio, sample_rate, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// The gradient operator for the SparseAdd op. -// -// The SparseAdd op calculates A + B, where A, B, and the sum are all represented -// as `SparseTensor` objects. This op takes in the upstream gradient w.r.t. -// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty -// values of A and B. -// -// Arguments: -// backprop_val_grad: 1-D with shape `[nnz(sum)]`. The gradient with respect to -// the non-empty values of the sum. -// a_indices: 2-D. The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`. -// b_indices: 2-D. The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`. -// sum_indices: 2-D. The `indices` of the sum `SparseTensor`, size -// `[nnz(sum), ndims]`. -// -// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the -// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the -// non-empty values of B. -func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SparseAddGrad", - Input: []tf.Input{ - backprop_val_grad, a_indices, b_indices, sum_indices, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - -// Read an element from the TensorArray into output `value`. -// -// Arguments: -// handle: The handle to a TensorArray. +// handle: The handle to a TensorArray. // // flow_in: A float scalar that enforces proper chaining of operations. // dtype: The type of the elem that is returned. @@ -13452,45 +13435,6 @@ func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, pa return op.Output(0) } -// L2 Loss. -// -// Computes half the L2 norm of a tensor without the `sqrt`: -// -// output = sum(t ** 2) / 2 -// -// Arguments: -// t: Typically 2-D, but may have any dimensions. -// -// Returns 0-D. -func L2Loss(scope *Scope, t tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "L2Loss", - Input: []tf.Input{ - t, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Computes rectified linear: `max(features, 0)`. -func Relu(scope *Scope, features tf.Output) (activations tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Relu", - Input: []tf.Input{ - features, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Returns the truth value of (x >= y) element-wise. // // *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting @@ -13594,35 +13538,6 @@ func Rint(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } -// Computes the Eigen Decomposition of a batch of square self-adjoint matrices. -// -// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead. -// -// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices, with the same constraints as the single matrix -// SelfAdjointEig. -// -// The result is a [..., M+1, M] matrix with [..., 0,:] containing the -// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. -// -// Arguments: -// input: Shape is `[..., M, M]`. -// -// Returns Shape is `[..., M+1, M]`. -func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SelfAdjointEig", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // QuantizeV2Attr is an optional argument to QuantizeV2. type QuantizeV2Attr func(optionalAttr) @@ -13634,11 +13549,21 @@ func QuantizeV2Mode(value string) QuantizeV2Attr { } } +// QuantizeV2RoundMode sets the optional round_mode attribute to value. +// If not specified, defaults to "HALF_AWAY_FROM_ZERO" +func QuantizeV2RoundMode(value string) QuantizeV2Attr { + return func(m optionalAttr) { + m["round_mode"] = value + } +} + // Quantize the 'input' tensor of type float to 'output' tensor of type 'T'. // // [min_range, max_range] are scalar floats that specify the range for // the 'input' data. The 'mode' attribute controls exactly which calculations are -// used to convert the float values to their quantized equivalents. +// used to convert the float values to their quantized equivalents. The +// 'round_mode' attribute controls which rounding tie-breaking algorithm is used +// when rounding float values to their quantized equivalents. // // In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: // @@ -13662,10 +13587,10 @@ func QuantizeV2Mode(value string) QuantizeV2Attr { // If the mode is 'MIN_FIRST', then this approach is used: // // ``` -// number_of_steps = 1 << (# of bits in T) -// range_adjust = number_of_steps / (number_of_steps - 1) +// num_discrete_values = 1 << (# of bits in T) +// range_adjust = num_discrete_values / (num_discrete_values - 1) // range = (range_max - range_min) * range_adjust -// range_scale = number_of_steps / range +// range_scale = num_discrete_values / range // quantized = round(input * range_scale) - round(range_min * range_scale) + // numeric_limits::min() // quantized = max(quantized, numeric_limits::min()) @@ -13715,7 +13640,7 @@ func QuantizeV2Mode(value string) QuantizeV2Attr { // // Now we can quantize the elements of our tensor: // ```c++ -// result = (input * s).round_to_nearest() +// result = round(input * s) // ``` // // One thing to watch out for is that the operator may choose to adjust the @@ -14263,7 +14188,7 @@ func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max // # [20, 21, 22, 23]]]] // # tensor 't' shape is [1, 2, 3, 4] // -// # 'dims' is [3] or 'dims' is -1 +// # 'dims' is [3] or 'dims' is [-1] // reverse(t, dims) ==> [[[[ 3, 2, 1, 0], // [ 7, 6, 5, 4], // [ 11, 10, 9, 8]], @@ -14402,6 +14327,32 @@ func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) { return op.Output(0) } +// Increments variable pointed to by 'resource' until it reaches 'limit'. +// +// Arguments: +// resource: Should be from a scalar `Variable` node. +// limit: If incrementing ref would bring it above limit, instead generates an +// 'OutOfRange' error. +// +// +// Returns A copy of the input before increment. If nothing else modifies the +// input, the values produced will all be distinct. +func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"limit": limit, "T": T} + opspec := tf.OpSpec{ + Type: "ResourceCountUpTo", + Input: []tf.Input{ + resource, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Looks up keys in a table, outputs the corresponding values. // // The tensor `keys` must of the same type as the keys of the table. @@ -14487,57 +14438,35 @@ func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, fil return op.Output(0) } -// Get the value of the tensor specified by its handle. -// -// Arguments: -// handle: The handle for a tensor stored in the session state. -// dtype: The type of the output value. -// -// Returns The tensor for the given handle. -func GetSessionTensor(scope *Scope, handle tf.Output, dtype tf.DataType) (value tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtype": dtype} - opspec := tf.OpSpec{ - Type: "GetSessionTensor", - Input: []tf.Input{ - handle, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// ResourceApplyProximalGradientDescentAttr is an optional argument to ResourceApplyProximalGradientDescent. -type ResourceApplyProximalGradientDescentAttr func(optionalAttr) +// MatrixSolveAttr is an optional argument to MatrixSolve. +type MatrixSolveAttr func(optionalAttr) -// ResourceApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value. +// MatrixSolveAdjoint sets the optional adjoint attribute to value. // -// value: If True, the subtraction will be protected by a lock; -// otherwise the behavior is undefined, but may exhibit less contention. +// value: Boolean indicating whether to solve with `matrix` or its (block-wise) +// adjoint. // If not specified, defaults to false -func ResourceApplyProximalGradientDescentUseLocking(value bool) ResourceApplyProximalGradientDescentAttr { +func MatrixSolveAdjoint(value bool) MatrixSolveAttr { return func(m optionalAttr) { - m["use_locking"] = value + m["adjoint"] = value } } -// Update '*var' as FOBOS algorithm with fixed learning rate. +// Solves systems of linear equations. // -// prox_v = var - alpha * delta -// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} +// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is +// a tensor shape `[..., M, K]`. If `adjoint` is `False` then each output matrix +// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`. +// If `adjoint` is `True` then each output matrix satisfies +// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`. // // Arguments: -// var_: Should be from a Variable(). -// alpha: Scaling factor. Must be a scalar. -// l1: L1 regularization. Must be a scalar. -// l2: L2 regularization. Must be a scalar. -// delta: The change. +// matrix: Shape is `[..., M, M]`. +// rhs: Shape is `[..., M, K]`. // -// Returns the created operation. -func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, delta tf.Output, optional ...ResourceApplyProximalGradientDescentAttr) (o *tf.Operation) { +// Returns Shape is `[..., M, K]`. +func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) { if scope.Err() != nil { return } @@ -14546,54 +14475,322 @@ func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceApplyProximalGradientDescent", + Type: "MatrixSolve", Input: []tf.Input{ - var_, alpha, l1, l2, delta, + matrix, rhs, }, Attrs: attrs, } - return scope.AddOperation(opspec) + op := scope.AddOperation(opspec) + return op.Output(0) } -// 2D fast Fourier transform. -// -// Computes the 2-dimensional discrete Fourier transform over the inner-most -// 2 dimensions of `input`. +// Transforms a Tensor into a serialized TensorProto proto. // // Arguments: -// input: A complex64 tensor. -// -// Returns A complex64 tensor of the same shape as `input`. The inner-most 2 -// dimensions of `input` are replaced with their 2D Fourier transform. +// tensor: A Tensor of type `T`. // -// @compatibility(numpy) -// Equivalent to np.fft.fft2 -// @end_compatibility -func FFT2D(scope *Scope, input tf.Output) (output tf.Output) { +// Returns A serialized TensorProto proto of the input tensor. +func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "FFT2D", + Type: "SerializeTensor", Input: []tf.Input{ - input, + tensor, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// Creates a tensor filled with a scalar value. +// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad. +type FusedBatchNormGradAttr func(optionalAttr) + +// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value. // -// This operation creates a tensor of shape `dims` and fills it with `value`. +// value: A small float number added to the variance of x. +// If not specified, defaults to 0.0001 +func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr { + return func(m optionalAttr) { + m["epsilon"] = value + } +} + +// FusedBatchNormGradDataFormat sets the optional data_format attribute to value. // -// For example: +// value: The data format for y_backprop, x, x_backprop. +// Either "NHWC" (default) or "NCHW". +// If not specified, defaults to "NHWC" +func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr { + return func(m optionalAttr) { + m["data_format"] = value + } +} + +// FusedBatchNormGradIsTraining sets the optional is_training attribute to value. // -// ``` -// # Output tensor has shape [2, 3]. -// fill([2, 3], 9) ==> [[9, 9, 9] -// [9, 9, 9]] -// ``` +// value: A bool value to indicate the operation is for training (default) +// or inference. +// If not specified, defaults to true +func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr { + return func(m optionalAttr) { + m["is_training"] = value + } +} + +// Gradient for batch normalization. +// +// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW". +// The size of 1D Tensors matches the dimension C of the 4D Tensors. +// +// Arguments: +// y_backprop: A 4D Tensor for the gradient with respect to y. +// x: A 4D Tensor for input data. +// scale: A 1D Tensor for scaling factor, to scale the normalized x. +// reserve_space_1: When is_training is True, a 1D Tensor for the computed batch +// mean to be reused in gradient computation. When is_training is +// False, a 1D Tensor for the population mean to be reused in both +// 1st and 2nd order gradient computation. +// reserve_space_2: When is_training is True, a 1D Tensor for the computed batch +// variance (inverted variance in the cuDNN case) to be reused in +// gradient computation. When is_training is False, a 1D Tensor +// for the population variance to be reused in both 1st and 2nd +// order gradient computation. +// +// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input +// in FusedBatchNorm. +func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradAttr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "FusedBatchNormGrad", + Input: []tf.Input{ + y_backprop, x, scale, reserve_space_1, reserve_space_2, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4) +} + +// Computes rectified linear: `max(features, 0)`. +func Relu(scope *Scope, features tf.Output) (activations tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Relu", + Input: []tf.Input{ + features, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// L2 Loss. +// +// Computes half the L2 norm of a tensor without the `sqrt`: +// +// output = sum(t ** 2) / 2 +// +// Arguments: +// t: Typically 2-D, but may have any dimensions. +// +// Returns 0-D. +func L2Loss(scope *Scope, t tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "L2Loss", + Input: []tf.Input{ + t, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ShapeAttr is an optional argument to Shape. +type ShapeAttr func(optionalAttr) + +// ShapeOutType sets the optional out_type attribute to value. +// If not specified, defaults to DT_INT32 +func ShapeOutType(value tf.DataType) ShapeAttr { + return func(m optionalAttr) { + m["out_type"] = value + } +} + +// Returns the shape of a tensor. +// +// This operation returns a 1-D integer tensor representing the shape of `input`. +// +// For example: +// +// ``` +// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]] +// shape(t) ==> [2, 2, 3] +// ``` +func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Shape", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes softmax cross entropy cost and gradients to backpropagate. +// +// Inputs are the logits, not probabilities. +// +// Arguments: +// features: batch_size x num_classes matrix +// labels: batch_size x num_classes matrix +// The caller must ensure that each batch of labels represents a valid +// probability distribution. +// +// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix). +func SoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SoftmaxCrossEntropyWithLogits", + Input: []tf.Input{ + features, labels, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + +// Get the value of the tensor specified by its handle. +// +// Arguments: +// handle: The handle for a tensor stored in the session state. +// dtype: The type of the output value. +// +// Returns The tensor for the given handle. +func GetSessionTensor(scope *Scope, handle tf.Output, dtype tf.DataType) (value tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtype": dtype} + opspec := tf.OpSpec{ + Type: "GetSessionTensor", + Input: []tf.Input{ + handle, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// ResourceApplyProximalGradientDescentAttr is an optional argument to ResourceApplyProximalGradientDescent. +type ResourceApplyProximalGradientDescentAttr func(optionalAttr) + +// ResourceApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value. +// +// value: If True, the subtraction will be protected by a lock; +// otherwise the behavior is undefined, but may exhibit less contention. +// If not specified, defaults to false +func ResourceApplyProximalGradientDescentUseLocking(value bool) ResourceApplyProximalGradientDescentAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// Update '*var' as FOBOS algorithm with fixed learning rate. +// +// prox_v = var - alpha * delta +// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0} +// +// Arguments: +// var_: Should be from a Variable(). +// alpha: Scaling factor. Must be a scalar. +// l1: L1 regularization. Must be a scalar. +// l2: L2 regularization. Must be a scalar. +// delta: The change. +// +// Returns the created operation. +func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, delta tf.Output, optional ...ResourceApplyProximalGradientDescentAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResourceApplyProximalGradientDescent", + Input: []tf.Input{ + var_, alpha, l1, l2, delta, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + +// 2D fast Fourier transform. +// +// Computes the 2-dimensional discrete Fourier transform over the inner-most +// 2 dimensions of `input`. +// +// Arguments: +// input: A complex64 tensor. +// +// Returns A complex64 tensor of the same shape as `input`. The inner-most 2 +// dimensions of `input` are replaced with their 2D Fourier transform. +// +// @compatibility(numpy) +// Equivalent to np.fft.fft2 +// @end_compatibility +func FFT2D(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "FFT2D", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Creates a tensor filled with a scalar value. +// +// This operation creates a tensor of shape `dims` and fills it with `value`. +// +// For example: +// +// ``` +// # Output tensor has shape [2, 3]. +// fill([2, 3], 9) ==> [[9, 9, 9] +// [9, 9, 9]] +// ``` // // Arguments: // dims: 1-D. Represents the shape of the output tensor. @@ -15868,7 +16065,7 @@ func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and // Returns x / y element-wise for integer types. // // Truncation designates that negative numbers will round fractional quantities -// toward zero. I.e. -7 / 5 = 1. This matches C semantics but it is different +// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different // than Python semantics. See `FloorDiv` for a division function that matches // Python Semantics. // @@ -17088,6 +17285,53 @@ func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Outp return op.Output(0) } +// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad. +type ResourceSparseApplyAdagradAttr func(optionalAttr) + +// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value. +// +// value: If `True`, updating of the var and accum tensors will be protected +// by a lock; otherwise the behavior is undefined, but may exhibit less +// contention. +// If not specified, defaults to false +func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr { + return func(m optionalAttr) { + m["use_locking"] = value + } +} + +// Update relevant entries in '*var' and '*accum' according to the adagrad scheme. +// +// That is for rows we have grad for, we update var and accum as follows: +// accum += grad * grad +// var -= lr * grad * (1 / sqrt(accum)) +// +// Arguments: +// var_: Should be from a Variable(). +// accum: Should be from a Variable(). +// lr: Learning rate. Must be a scalar. +// grad: The gradient. +// indices: A vector of indices into the first dimension of var and accum. +// +// Returns the created operation. +func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResourceSparseApplyAdagrad", + Input: []tf.Input{ + var_, accum, lr, grad, indices, + }, + Attrs: attrs, + } + return scope.AddOperation(opspec) +} + // Creates a dataset that zips together `input_datasets`. func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { if scope.Err() != nil { @@ -17239,79 +17483,31 @@ func LRNAlpha(value float32) LRNAttr { // LRNBeta sets the optional beta attribute to value. // -// value: An exponent. -// If not specified, defaults to 0.5 -func LRNBeta(value float32) LRNAttr { - return func(m optionalAttr) { - m["beta"] = value - } -} - -// Local Response Normalization. -// -// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last -// dimension), and each vector is normalized independently. Within a given vector, -// each component is divided by the weighted, squared sum of inputs within -// `depth_radius`. In detail, -// -// sqr_sum[a, b, c, d] = -// sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2) -// output = input / (bias + alpha * sqr_sum) ** beta -// -// For details, see [Krizhevsky et al., ImageNet classification with deep -// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks). -// -// Arguments: -// input: 4-D. -func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "LRN", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad. -type ResourceSparseApplyAdagradAttr func(optionalAttr) - -// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value. -// -// value: If `True`, updating of the var and accum tensors will be protected -// by a lock; otherwise the behavior is undefined, but may exhibit less -// contention. -// If not specified, defaults to false -func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr { +// value: An exponent. +// If not specified, defaults to 0.5 +func LRNBeta(value float32) LRNAttr { return func(m optionalAttr) { - m["use_locking"] = value + m["beta"] = value } } -// Update relevant entries in '*var' and '*accum' according to the adagrad scheme. +// Local Response Normalization. // -// That is for rows we have grad for, we update var and accum as follows: -// accum += grad * grad -// var -= lr * grad * (1 / sqrt(accum)) +// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last +// dimension), and each vector is normalized independently. Within a given vector, +// each component is divided by the weighted, squared sum of inputs within +// `depth_radius`. In detail, // -// Arguments: -// var_: Should be from a Variable(). -// accum: Should be from a Variable(). -// lr: Learning rate. Must be a scalar. -// grad: The gradient. -// indices: A vector of indices into the first dimension of var and accum. +// sqr_sum[a, b, c, d] = +// sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2) +// output = input / (bias + alpha * sqr_sum) ** beta // -// Returns the created operation. -func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) { +// For details, see [Krizhevsky et al., ImageNet classification with deep +// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks). +// +// Arguments: +// input: 4-D. +func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) { if scope.Err() != nil { return } @@ -17320,13 +17516,14 @@ func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, l a(attrs) } opspec := tf.OpSpec{ - Type: "ResourceSparseApplyAdagrad", + Type: "LRN", Input: []tf.Input{ - var_, accum, lr, grad, indices, + input, }, Attrs: attrs, } - return scope.AddOperation(opspec) + op := scope.AddOperation(opspec) + return op.Output(0) } // AvgPool3DGradAttr is an optional argument to AvgPool3DGrad. @@ -17705,6 +17902,67 @@ func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64, return op.Output(0) } +// Encode audio data using the WAV file format. +// +// This operation will generate a string suitable to be saved out to create a .wav +// audio file. It will be encoded in the 16-bit PCM format. It takes in float +// values in the range -1.0f to 1.0f, and any outside that value will be clamped to +// that range. +// +// `audio` is a 2-D float Tensor of shape `[length, channels]`. +// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100). +// +// Arguments: +// audio: 2-D with shape `[length, channels]`. +// sample_rate: Scalar containing the sample frequency. +// +// Returns 0-D. WAV-encoded file contents. +func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "EncodeWav", + Input: []tf.Input{ + audio, sample_rate, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// The gradient operator for the SparseAdd op. +// +// The SparseAdd op calculates A + B, where A, B, and the sum are all represented +// as `SparseTensor` objects. This op takes in the upstream gradient w.r.t. +// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty +// values of A and B. +// +// Arguments: +// backprop_val_grad: 1-D with shape `[nnz(sum)]`. The gradient with respect to +// the non-empty values of the sum. +// a_indices: 2-D. The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`. +// b_indices: 2-D. The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`. +// sum_indices: 2-D. The `indices` of the sum `SparseTensor`, size +// `[nnz(sum), ndims]`. +// +// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the +// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the +// non-empty values of B. +func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SparseAddGrad", + Input: []tf.Input{ + backprop_val_grad, a_indices, b_indices, sum_indices, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + // FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2. type FixedLengthRecordReaderV2Attr func(optionalAttr) @@ -18019,124 +18277,313 @@ func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) { return } opspec := tf.OpSpec{ - Type: "Pad", + Type: "Pad", + Input: []tf.Input{ + input, paddings, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes the number of elements in the given queue. +// +// Arguments: +// handle: The handle to a queue. +// +// Returns The number of elements in the given queue. +func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "QueueSizeV2", + Input: []tf.Input{ + handle, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Outputs a `Summary` protocol buffer with a histogram. +// +// The generated +// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) +// has one summary value containing a histogram for `values`. +// +// This op reports an `InvalidArgument` error if any value is not finite. +// +// Arguments: +// tag: Scalar. Tag to use for the `Summary.Value`. +// values: Any shape. Values to use to build the histogram. +// +// Returns Scalar. Serialized `Summary` protocol buffer. +func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "HistogramSummary", + Input: []tf.Input{ + tag, values, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// AsStringAttr is an optional argument to AsString. +type AsStringAttr func(optionalAttr) + +// AsStringPrecision sets the optional precision attribute to value. +// +// value: The post-decimal precision to use for floating point numbers. +// Only used if precision > -1. +// If not specified, defaults to -1 +func AsStringPrecision(value int64) AsStringAttr { + return func(m optionalAttr) { + m["precision"] = value + } +} + +// AsStringScientific sets the optional scientific attribute to value. +// +// value: Use scientific notation for floating point numbers. +// If not specified, defaults to false +func AsStringScientific(value bool) AsStringAttr { + return func(m optionalAttr) { + m["scientific"] = value + } +} + +// AsStringShortest sets the optional shortest attribute to value. +// +// value: Use shortest representation (either scientific or standard) for +// floating point numbers. +// If not specified, defaults to false +func AsStringShortest(value bool) AsStringAttr { + return func(m optionalAttr) { + m["shortest"] = value + } +} + +// AsStringWidth sets the optional width attribute to value. +// +// value: Pad pre-decimal numbers to this width. +// Applies to both floating point and integer numbers. +// Only used if width > -1. +// If not specified, defaults to -1 +func AsStringWidth(value int64) AsStringAttr { + return func(m optionalAttr) { + m["width"] = value + } +} + +// AsStringFill sets the optional fill attribute to value. +// +// value: The value to pad if width > -1. If empty, pads with spaces. +// Another typical value is '0'. String cannot be longer than 1 character. +// If not specified, defaults to "" +func AsStringFill(value string) AsStringAttr { + return func(m optionalAttr) { + m["fill"] = value + } +} + +// Converts each entry in the given tensor to strings. Supports many numeric +// +// types and boolean. +func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "AsString", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Assigns sparse updates to the variable referenced by `resource`. +// +// This operation computes +// +// # Scalar indices +// ref[indices, ...] = updates[...] +// +// # Vector indices (for each i) +// ref[indices[i], ...] = updates[i, ...] +// +// # High rank indices (for each i, ..., j) +// ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] +// +// Arguments: +// resource: Should be from a `Variable` node. +// indices: A tensor of indices into the first dimension of `ref`. +// updates: A tensor of updated values to add to `ref`. +// +// Returns the created operation. +func ResourceScatterUpdate(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "ResourceScatterUpdate", Input: []tf.Input{ - input, paddings, + resource, indices, updates, }, } - op := scope.AddOperation(opspec) - return op.Output(0) + return scope.AddOperation(opspec) } -// Computes the number of elements in the given queue. +// Given a path to new and old vocabulary files, returns a remapping Tensor of +// +// length `num_new_vocab`, where `remapping[i]` contains the row number in the old +// vocabulary that corresponds to row `i` in the new vocabulary (starting at line +// `new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i` +// in the new vocabulary is not in the old vocabulary. `num_vocab_offset` enables +// use in the partitioned variable case, and should generally be set through +// examining partitioning info. The format of the files should be a text file, +// with each line containing a single entity within the vocabulary. +// +// For example, with `new_vocab_file` a text file containing each of the following +// elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3], +// `num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be +// `[0, -1, 2]`. +// +// The op also returns a count of how many entries in the new vocabulary +// were present in the old vocabulary, which is used to calculate the number of +// values to initialize in a weight matrix remapping +// +// This functionality can be used to remap both row vocabularies (typically, +// features) and column vocabularies (typically, classes) from TensorFlow +// checkpoints. Note that the partitioning logic relies on contiguous vocabularies +// corresponding to div-partitioned variables. Moreover, the underlying remapping +// uses an IndexTable (as opposed to an inexact CuckooTable), so client code should +// use the corresponding index_table_from_file() as the FeatureColumn framework +// does (as opposed to tf.feature_to_id(), which uses a CuckooTable). // // Arguments: -// handle: The handle to a queue. +// new_vocab_file: Path to the new vocab file. +// old_vocab_file: Path to the old vocab file. +// new_vocab_offset: How many entries into the new vocab file to start reading. +// num_new_vocab: Number of entries in the new vocab file to remap. // -// Returns The number of elements in the given queue. -func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) { +// Returns A Tensor of length num_new_vocab where the element at index i +// is equal to the old ID that maps to the new ID i. This element is -1 for any +// new ID that is not found in the old vocabulary.Number of new vocab entries found in old vocab. +func GenerateVocabRemapping(scope *Scope, new_vocab_file tf.Output, old_vocab_file tf.Output, new_vocab_offset int64, num_new_vocab int64) (remapping tf.Output, num_present tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{"new_vocab_offset": new_vocab_offset, "num_new_vocab": num_new_vocab} opspec := tf.OpSpec{ - Type: "QueueSizeV2", + Type: "GenerateVocabRemapping", Input: []tf.Input{ - handle, + new_vocab_file, old_vocab_file, }, + Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) + return op.Output(0), op.Output(1) } -// Outputs a `Summary` protocol buffer with a histogram. -// -// The generated -// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) -// has one summary value containing a histogram for `values`. -// -// This op reports an `InvalidArgument` error if any value is not finite. -// -// Arguments: -// tag: Scalar. Tag to use for the `Summary.Value`. -// values: Any shape. Values to use to build the histogram. -// -// Returns Scalar. Serialized `Summary` protocol buffer. -func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) { +// Computes softsign: `features / (abs(features) + 1)`. +func Softsign(scope *Scope, features tf.Output) (activations tf.Output) { if scope.Err() != nil { return } opspec := tf.OpSpec{ - Type: "HistogramSummary", + Type: "Softsign", Input: []tf.Input{ - tag, values, + features, }, } op := scope.AddOperation(opspec) return op.Output(0) } -// AsStringAttr is an optional argument to AsString. -type AsStringAttr func(optionalAttr) +// ResizeBilinearAttr is an optional argument to ResizeBilinear. +type ResizeBilinearAttr func(optionalAttr) -// AsStringPrecision sets the optional precision attribute to value. +// ResizeBilinearAlignCorners sets the optional align_corners attribute to value. // -// value: The post-decimal precision to use for floating point numbers. -// Only used if precision > -1. -// If not specified, defaults to -1 -func AsStringPrecision(value int64) AsStringAttr { +// value: If true, rescale input by (new_height - 1) / (height - 1), which +// exactly aligns the 4 corners of images and resized images. If false, rescale +// by new_height / height. Treat similarly the width dimension. +// If not specified, defaults to false +func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr { return func(m optionalAttr) { - m["precision"] = value + m["align_corners"] = value } } -// AsStringScientific sets the optional scientific attribute to value. +// Resize `images` to `size` using bilinear interpolation. // -// value: Use scientific notation for floating point numbers. -// If not specified, defaults to false -func AsStringScientific(value bool) AsStringAttr { - return func(m optionalAttr) { - m["scientific"] = value +// Input images can be of different types but output images are always float. +// +// Arguments: +// images: 4-D with shape `[batch, height, width, channels]`. +// size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`. The +// new size for the images. +// +// Returns 4-D with shape +// `[batch, new_height, new_width, channels]`. +func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "ResizeBilinear", + Input: []tf.Input{ + images, size, + }, + Attrs: attrs, } + op := scope.AddOperation(opspec) + return op.Output(0) } -// AsStringShortest sets the optional shortest attribute to value. +// ProdAttr is an optional argument to Prod. +type ProdAttr func(optionalAttr) + +// ProdKeepDims sets the optional keep_dims attribute to value. // -// value: Use shortest representation (either scientific or standard) for -// floating point numbers. +// value: If true, retain reduced dimensions with length 1. // If not specified, defaults to false -func AsStringShortest(value bool) AsStringAttr { +func ProdKeepDims(value bool) ProdAttr { return func(m optionalAttr) { - m["shortest"] = value + m["keep_dims"] = value } } -// AsStringWidth sets the optional width attribute to value. +// Computes the product of elements across dimensions of a tensor. // -// value: Pad pre-decimal numbers to this width. -// Applies to both floating point and integer numbers. -// Only used if width > -1. -// If not specified, defaults to -1 -func AsStringWidth(value int64) AsStringAttr { - return func(m optionalAttr) { - m["width"] = value - } -} - -// AsStringFill sets the optional fill attribute to value. +// Reduces `input` along the dimensions given in `reduction_indices`. Unless +// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in +// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are +// retained with length 1. // -// value: The value to pad if width > -1. If empty, pads with spaces. -// Another typical value is '0'. String cannot be longer than 1 character. -// If not specified, defaults to "" -func AsStringFill(value string) AsStringAttr { - return func(m optionalAttr) { - m["fill"] = value - } -} - -// Converts each entry in the given tensor to strings. Supports many numeric +// Arguments: +// input: The tensor to reduce. +// reduction_indices: The dimensions to reduce. Must be in the range +// `[-rank(input), rank(input))`. // -// types and boolean. -func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output tf.Output) { +// Returns The reduced tensor. +func Prod(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...ProdAttr) (output tf.Output) { if scope.Err() != nil { return } @@ -18145,9 +18592,9 @@ func AsString(scope *Scope, input tf.Output, optional ...AsStringAttr) (output t a(attrs) } opspec := tf.OpSpec{ - Type: "AsString", + Type: "Prod", Input: []tf.Input{ - input, + input, reduction_indices, }, Attrs: attrs, } @@ -20443,150 +20890,72 @@ func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr { // input tensor. // padding: The type of padding algorithm to use. // -// Returns The max pooled output tensor.4-D. The flattened indices of the max values chosen for each output. -func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MaxPoolWithArgmax", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - -// MaxPoolGradGradV2Attr is an optional argument to MaxPoolGradGradV2. -type MaxPoolGradGradV2Attr func(optionalAttr) - -// MaxPoolGradGradV2DataFormat sets the optional data_format attribute to value. -// -// value: Specify the data format of the input and output data. With the -// default format "NHWC", the data is stored in the order of: -// [batch, in_height, in_width, in_channels]. -// Alternatively, the format could be "NCHW", the data storage order of: -// [batch, in_channels, in_height, in_width]. -// If not specified, defaults to "NHWC" -func MaxPoolGradGradV2DataFormat(value string) MaxPoolGradGradV2Attr { - return func(m optionalAttr) { - m["data_format"] = value - } -} - -// Computes second-order gradients of the maxpooling function. -// -// Arguments: -// orig_input: The original input tensor. -// orig_output: The original output tensor. -// grad: 4-D. Gradients of gradients w.r.t. the input of `max_pool`. -// ksize: The size of the window for each dimension of the input tensor. -// strides: The stride of the sliding window for each dimension of the -// input tensor. -// padding: The type of padding algorithm to use. -// -// Returns Gradients of gradients w.r.t. the input to `max_pool`. -func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradGradV2Attr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"padding": padding} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MaxPoolGradGradV2", - Input: []tf.Input{ - orig_input, orig_output, grad, ksize, strides, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Adjust the saturation of one or more images. -// -// `images` is a tensor of at least 3 dimensions. The last dimension is -// interpretted as channels, and must be three. -// -// The input image is considered in the RGB colorspace. Conceptually, the RGB -// colors are first mapped into HSV. A scale is then applied all the saturation -// values, and then remapped back to RGB colorspace. -// -// Arguments: -// images: Images to adjust. At least 3-D. -// scale: A float scale to add to the saturation. -// -// Returns The hue-adjusted image or images. -func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) { +// Returns The max pooled output tensor.4-D. The flattened indices of the max values chosen for each output. +func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) { if scope.Err() != nil { return } + attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ - Type: "AdjustSaturation", + Type: "MaxPoolWithArgmax", Input: []tf.Input{ - images, scale, + input, }, + Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0) + return op.Output(0), op.Output(1) } -// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2. -type SelfAdjointEigV2Attr func(optionalAttr) +// MaxPoolGradGradV2Attr is an optional argument to MaxPoolGradGradV2. +type MaxPoolGradGradV2Attr func(optionalAttr) -// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value. +// MaxPoolGradGradV2DataFormat sets the optional data_format attribute to value. // -// value: If `True` then eigenvectors will be computed and returned in `v`. -// Otherwise, only the eigenvalues will be computed. -// If not specified, defaults to true -func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr { +// value: Specify the data format of the input and output data. With the +// default format "NHWC", the data is stored in the order of: +// [batch, in_height, in_width, in_channels]. +// Alternatively, the format could be "NCHW", the data storage order of: +// [batch, in_channels, in_height, in_width]. +// If not specified, defaults to "NHWC" +func MaxPoolGradGradV2DataFormat(value string) MaxPoolGradGradV2Attr { return func(m optionalAttr) { - m["compute_v"] = value + m["data_format"] = value } } -// Computes the eigen decomposition of one or more square self-adjoint matrices. -// -// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in -// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. -// -// ```python -// # a is a tensor. -// # e is a tensor of eigenvalues. -// # v is a tensor of eigenvectors. -// e, v = self_adjoint_eig(a) -// e = self_adjoint_eig(a, compute_v=False) -// ``` +// Computes second-order gradients of the maxpooling function. // // Arguments: -// input: `Tensor` input of shape `[N, N]`. +// orig_input: The original input tensor. +// orig_output: The original output tensor. +// grad: 4-D. Gradients of gradients w.r.t. the input of `max_pool`. +// ksize: The size of the window for each dimension of the input tensor. +// strides: The stride of the sliding window for each dimension of the +// input tensor. +// padding: The type of padding algorithm to use. // -// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`. -func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) { +// Returns Gradients of gradients w.r.t. the input to `max_pool`. +func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradGradV2Attr) (output tf.Output) { if scope.Err() != nil { return } - attrs := map[string]interface{}{} + attrs := map[string]interface{}{"padding": padding} for _, a := range optional { a(attrs) } opspec := tf.OpSpec{ - Type: "SelfAdjointEigV2", + Type: "MaxPoolGradGradV2", Input: []tf.Input{ - input, + orig_input, orig_output, grad, ksize, strides, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) + return op.Output(0) } // Computes second-order gradients of the maxpooling function. @@ -21257,163 +21626,6 @@ func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backpr return op.Output(0) } -// Given a path to new and old vocabulary files, returns a remapping Tensor of -// -// length `num_new_vocab`, where `remapping[i]` contains the row number in the old -// vocabulary that corresponds to row `i` in the new vocabulary (starting at line -// `new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i` -// in the new vocabulary is not in the old vocabulary. `num_vocab_offset` enables -// use in the partitioned variable case, and should generally be set through -// examining partitioning info. The format of the files should be a text file, -// with each line containing a single entity within the vocabulary. -// -// For example, with `new_vocab_file` a text file containing each of the following -// elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3], -// `num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be -// `[0, -1, 2]`. -// -// The op also returns a count of how many entries in the new vocabulary -// were present in the old vocabulary, which is used to calculate the number of -// values to initialize in a weight matrix remapping -// -// This functionality can be used to remap both row vocabularies (typically, -// features) and column vocabularies (typically, classes) from TensorFlow -// checkpoints. Note that the partitioning logic relies on contiguous vocabularies -// corresponding to div-partitioned variables. Moreover, the underlying remapping -// uses an IndexTable (as opposed to an inexact CuckooTable), so client code should -// use the corresponding index_table_from_file() as the FeatureColumn framework -// does (as opposed to tf.feature_to_id(), which uses a CuckooTable). -// -// Arguments: -// new_vocab_file: Path to the new vocab file. -// old_vocab_file: Path to the old vocab file. -// new_vocab_offset: How many entries into the new vocab file to start reading. -// num_new_vocab: Number of entries in the new vocab file to remap. -// -// Returns A Tensor of length num_new_vocab where the element at index i -// is equal to the old ID that maps to the new ID i. This element is -1 for any -// new ID that is not found in the old vocabulary.Number of new vocab entries found in old vocab. -func GenerateVocabRemapping(scope *Scope, new_vocab_file tf.Output, old_vocab_file tf.Output, new_vocab_offset int64, num_new_vocab int64) (remapping tf.Output, num_present tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"new_vocab_offset": new_vocab_offset, "num_new_vocab": num_new_vocab} - opspec := tf.OpSpec{ - Type: "GenerateVocabRemapping", - Input: []tf.Input{ - new_vocab_file, old_vocab_file, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - -// Computes softsign: `features / (abs(features) + 1)`. -func Softsign(scope *Scope, features tf.Output) (activations tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Softsign", - Input: []tf.Input{ - features, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// ResizeBilinearAttr is an optional argument to ResizeBilinear. -type ResizeBilinearAttr func(optionalAttr) - -// ResizeBilinearAlignCorners sets the optional align_corners attribute to value. -// -// value: If true, rescale input by (new_height - 1) / (height - 1), which -// exactly aligns the 4 corners of images and resized images. If false, rescale -// by new_height / height. Treat similarly the width dimension. -// If not specified, defaults to false -func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr { - return func(m optionalAttr) { - m["align_corners"] = value - } -} - -// Resize `images` to `size` using bilinear interpolation. -// -// Input images can be of different types but output images are always float. -// -// Arguments: -// images: 4-D with shape `[batch, height, width, channels]`. -// size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`. The -// new size for the images. -// -// Returns 4-D with shape -// `[batch, new_height, new_width, channels]`. -func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "ResizeBilinear", - Input: []tf.Input{ - images, size, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// ProdAttr is an optional argument to Prod. -type ProdAttr func(optionalAttr) - -// ProdKeepDims sets the optional keep_dims attribute to value. -// -// value: If true, retain reduced dimensions with length 1. -// If not specified, defaults to false -func ProdKeepDims(value bool) ProdAttr { - return func(m optionalAttr) { - m["keep_dims"] = value - } -} - -// Computes the product of elements across dimensions of a tensor. -// -// Reduces `input` along the dimensions given in `reduction_indices`. Unless -// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in -// `reduction_indices`. If `keep_dims` is true, the reduced dimensions are -// retained with length 1. -// -// Arguments: -// input: The tensor to reduce. -// reduction_indices: The dimensions to reduce. Must be in the range -// `[-rank(input), rank(input))`. -// -// Returns The reduced tensor. -func Prod(scope *Scope, input tf.Output, reduction_indices tf.Output, optional ...ProdAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "Prod", - Input: []tf.Input{ - input, reduction_indices, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // Computes softsign gradients for a softsign operation. // // Arguments: @@ -22201,12 +22413,68 @@ func QuantizedBiasAdd(scope *Scope, input tf.Output, bias tf.Output, min_input t opspec := tf.OpSpec{ Type: "QuantizedBiasAdd", Input: []tf.Input{ - input, bias, min_input, max_input, min_bias, max_bias, + input, bias, min_input, max_input, min_bias, max_bias, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} + +// HistogramFixedWidthAttr is an optional argument to HistogramFixedWidth. +type HistogramFixedWidthAttr func(optionalAttr) + +// HistogramFixedWidthDtype sets the optional dtype attribute to value. +// If not specified, defaults to DT_INT32 +func HistogramFixedWidthDtype(value tf.DataType) HistogramFixedWidthAttr { + return func(m optionalAttr) { + m["dtype"] = value + } +} + +// Return histogram of values. +// +// Given the tensor `values`, this operation returns a rank 1 histogram counting +// the number of entries in `values` that fall into every bin. The bins are +// equal width and determined by the arguments `value_range` and `nbins`. +// +// ```python +// # Bins will be: (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf) +// nbins = 5 +// value_range = [0.0, 5.0] +// new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] +// +// with tf.get_default_session() as sess: +// hist = tf.histogram_fixed_width(new_values, value_range, nbins=5) +// variables.global_variables_initializer().run() +// sess.run(hist) => [2, 1, 1, 0, 2] +// ``` +// +// Arguments: +// values: Numeric `Tensor`. +// value_range: Shape [2] `Tensor` of same `dtype` as `values`. +// values <= value_range[0] will be mapped to hist[0], +// values >= value_range[1] will be mapped to hist[-1]. +// nbins: Scalar `int32 Tensor`. Number of histogram bins. +// +// Returns A 1-D `Tensor` holding histogram of values. +func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output, nbins tf.Output, optional ...HistogramFixedWidthAttr) (out tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "HistogramFixedWidth", + Input: []tf.Input{ + values, value_range, nbins, }, Attrs: attrs, } op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) + return op.Output(0) } // Quantized Batch normalization. @@ -22606,6 +22874,76 @@ func Sqrt(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// MatrixInverseAttr is an optional argument to MatrixInverse. +type MatrixInverseAttr func(optionalAttr) + +// MatrixInverseAdjoint sets the optional adjoint attribute to value. +// If not specified, defaults to false +func MatrixInverseAdjoint(value bool) MatrixInverseAttr { + return func(m optionalAttr) { + m["adjoint"] = value + } +} + +// Computes the inverse of one or more square invertible matrices or their +// +// adjoints (conjugate transposes). +// +// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions +// form square matrices. The output is a tensor of the same shape as the input +// containing the inverse for all input submatrices `[..., :, :]`. +// +// The op uses LU decomposition with partial pivoting to compute the inverses. +// +// If a matrix is not invertible there is no guarantee what the op does. It +// may detect the condition and raise an exception or it may simply return a +// garbage result. +// +// Arguments: +// input: Shape is `[..., M, M]`. +// +// Returns Shape is `[..., M, M]`. +// +// @compatibility(numpy) +// Equivalent to np.linalg.inv +// @end_compatibility +func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MatrixInverse", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Computes the gradient for the sqrt of `x` wrt its input. +// +// Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy` +// is the corresponding input gradient. +func SqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "SqrtGrad", + Input: []tf.Input{ + y, dy, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Inserts a dimension of 1 into a tensor's shape. // // Given a tensor `input`, this operation inserts a dimension of 1 at the @@ -23667,6 +24005,55 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu return op.Output(0) } +// NthElementAttr is an optional argument to NthElement. +type NthElementAttr func(optionalAttr) + +// NthElementReverse sets the optional reverse attribute to value. +// +// value: When set to True, find the nth-largest value in the vector and vice +// versa. +// If not specified, defaults to false +func NthElementReverse(value bool) NthElementAttr { + return func(m optionalAttr) { + m["reverse"] = value + } +} + +// Finds values of the `n`-th order statistic for the last dmension. +// +// If the input is a vector (rank-1), finds the entries which is the nth-smallest +// value in the vector and outputs their values as scalar tensor. +// +// For matrices (resp. higher rank input), computes the entries which is the +// nth-smallest value in each row (resp. vector along the last dimension). Thus, +// +// values.shape = input.shape[:-1] +// +// Arguments: +// input: 1-D or higher with last dimension at least `n+1`. +// n: 0-D. Position of sorted vector to select along the last dimension (along +// each row for matrices). Valid range of n is `[0, input.shape[:-1])` +// +// Returns The `n`-th order statistic along each last dimensional slice. +func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "NthElement", + Input: []tf.Input{ + input, n, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Computes asin of x element-wise. func Asin(scope *Scope, x tf.Output) (y tf.Output) { if scope.Err() != nil { @@ -24110,6 +24497,24 @@ func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { return op.Output(0) } +// Returns x + y element-wise. +// +// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +func AddV2(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "AddV2", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Saves the input tensors to disk. // // The size of `tensor_names` must match the number of tensors in `data`. `data[i]` @@ -26700,140 +27105,3 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t op := scope.AddOperation(opspec) return op.Output(0) } - -// Computes the gradient for the sqrt of `x` wrt its input. -// -// Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy` -// is the corresponding input gradient. -func SqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SqrtGrad", - Input: []tf.Input{ - y, dy, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// MatrixInverseAttr is an optional argument to MatrixInverse. -type MatrixInverseAttr func(optionalAttr) - -// MatrixInverseAdjoint sets the optional adjoint attribute to value. -// If not specified, defaults to false -func MatrixInverseAdjoint(value bool) MatrixInverseAttr { - return func(m optionalAttr) { - m["adjoint"] = value - } -} - -// Computes the inverse of one or more square invertible matrices or their -// -// adjoints (conjugate transposes). -// -// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices. The output is a tensor of the same shape as the input -// containing the inverse for all input submatrices `[..., :, :]`. -// -// The op uses LU decomposition with partial pivoting to compute the inverses. -// -// If a matrix is not invertible there is no guarantee what the op does. It -// may detect the condition and raise an exception or it may simply return a -// garbage result. -// -// Arguments: -// input: Shape is `[..., M, M]`. -// -// Returns Shape is `[..., M, M]`. -// -// @compatibility(numpy) -// Equivalent to np.linalg.inv -// @end_compatibility -func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MatrixInverse", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Transforms a Tensor into a serialized TensorProto proto. -// -// Arguments: -// tensor: A Tensor of type `T`. -// -// Returns A serialized TensorProto proto of the input tensor. -func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "SerializeTensor", - Input: []tf.Input{ - tensor, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// MatrixSolveAttr is an optional argument to MatrixSolve. -type MatrixSolveAttr func(optionalAttr) - -// MatrixSolveAdjoint sets the optional adjoint attribute to value. -// -// value: Boolean indicating whether to solve with `matrix` or its (block-wise) -// adjoint. -// If not specified, defaults to false -func MatrixSolveAdjoint(value bool) MatrixSolveAttr { - return func(m optionalAttr) { - m["adjoint"] = value - } -} - -// Solves systems of linear equations. -// -// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions -// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is -// a tensor shape `[..., M, K]`. If `adjoint` is `False` then each output matrix -// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`. -// If `adjoint` is `True` then each output matrix satisfies -// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`. -// -// Arguments: -// matrix: Shape is `[..., M, M]`. -// rhs: Shape is `[..., M, K]`. -// -// Returns Shape is `[..., M, K]`. -func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MatrixSolve", - Input: []tf.Input{ - matrix, rhs, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD index ea6c5a494a5bd8aa65cd53ca83f8fbae5e567b14..c0563da06d99bcf06477c094b560ceff6a01eff0 100644 --- a/tensorflow/java/BUILD +++ b/tensorflow/java/BUILD @@ -280,20 +280,20 @@ tf_java_test( ], ) -java_test( - name = "OperatorProcessorTest", - size = "small", - srcs = ["src/test/java/org/tensorflow/processor/OperatorProcessorTest.java"], - javacopts = JAVACOPTS, - resources = [":processor_test_resources"], - test_class = "org.tensorflow.processor.OperatorProcessorTest", - deps = [ - ":processor_library", - "@com_google_testing_compile", - "@com_google_truth", - "@junit", - ], -) +#java_test( +# name = "OperatorProcessorTest", +# size = "small", +# srcs = ["src/test/java/org/tensorflow/processor/OperatorProcessorTest.java"], +# javacopts = JAVACOPTS, +# resources = [":processor_test_resources"], +# test_class = "org.tensorflow.processor.OperatorProcessorTest", +# deps = [ +# ":processor_library", +# "//third_party/java/junit", +# "@com_google_testing_compile", +# "@com_google_truth", +# ], +#) filegroup( name = "processor_test_resources", diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml index a5b05132cfdd4b4d5c7a25e5d0a294b0c9187af1..371457087616b43ae81eb3b72b46b189f2ff5c19 100644 --- a/tensorflow/java/maven/libtensorflow/pom.xml +++ b/tensorflow/java/maven/libtensorflow/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.4.0-rc0 + 1.4.0-rc1 ../ libtensorflow diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml index d863f03e3c761aa6c31ac6cda439ce87b51ca637..9f7eb402530bf57fa8745b47696920b51d9fe5e5 100644 --- a/tensorflow/java/maven/libtensorflow_jni/pom.xml +++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.4.0-rc0 + 1.4.0-rc1 ../ libtensorflow_jni diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml index 1f5b0569615fac4d30e8f31eb550b2042c687512..fac0a8bc260f1a502d905f743225a4963209dbaa 100644 --- a/tensorflow/java/maven/pom.xml +++ b/tensorflow/java/maven/pom.xml @@ -6,7 +6,7 @@ 4.0.0 org.tensorflow parentpom - 1.4.0-rc0 + 1.4.0-rc1 pom https://www.tensorflow.org diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml index 51d53f6aba9c750d4d96d362290dac23a7b3e668..135ee0f2d2a3954fdf5789545093c209ba2fad0c 100644 --- a/tensorflow/java/maven/proto/pom.xml +++ b/tensorflow/java/maven/proto/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.4.0-rc0 + 1.4.0-rc1 ../ proto diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml index 8cc3f113e386d2e10ee21ae0825f4c9b119eff22..771482ba641167484482cbbcac83f22e56ff728d 100644 --- a/tensorflow/java/maven/tensorflow/pom.xml +++ b/tensorflow/java/maven/tensorflow/pom.xml @@ -6,7 +6,7 @@ org.tensorflow parentpom - 1.4.0-rc0 + 1.4.0-rc1 ../ tensorflow diff --git a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java index 19b4f8dddadbb23ccfe5065e00e6b5a71d7209c7..45e42878c770b3c19d96790e5b4bf2ed41a0de29 100644 --- a/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java +++ b/tensorflow/java/src/gen/java/org/tensorflow/processor/OperatorProcessor.java @@ -1,3 +1,18 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + package org.tensorflow.processor; import java.io.IOException; diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java index 5ad324263712ffa8254012389a113f7c8bffea84..7d12857dfaa2f4dc2c8fa4ea5307a00c974b14b8 100644 --- a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java +++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/bad/BasicBad.java @@ -1,3 +1,18 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + package org.tensorflow.processor.operator.bad; import org.tensorflow.op.annotation.Operator; diff --git a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java index fb69e8393945921528a83b65f236398d4b4dcc0e..4cf175f00de846447e20e02c91b3cd45dc80c750 100644 --- a/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java +++ b/tensorflow/java/src/test/resources/org/tensorflow/processor/operator/good/BasicGood.java @@ -1,3 +1,18 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + package org.tensorflow.processor.operator.good; import org.tensorflow.op.annotation.Operator; diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 1885caf695397e7801b48faf9c0880499f58e87b..02e88f4888f7813162894c24770b932b49cd454a 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -245,6 +245,17 @@ cc_library( ], ) +cc_library( + name = "cpp_python_util", + srcs = ["util/util.cc"], + hdrs = ["util/util.h"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//util/python:python_headers", + ], +) + cc_library( name = "py_func_lib", srcs = ["lib/core/py_func.cc"], @@ -886,6 +897,7 @@ py_library( ":tensor_shape", ":util", ":variable_scope", + "//tensorflow/python/eager:context", ], ) @@ -1141,6 +1153,7 @@ py_test( ":client_testlib", ":framework", ":framework_for_generated_wrappers", + ":framework_test_lib", ":math_ops", ":state_ops_gen", "//third_party/py/numpy", @@ -1909,6 +1922,7 @@ py_library( ":array_ops", ":control_flow_ops", ":framework_for_generated_wrappers", + "//tensorflow/python/eager:context", ], ) @@ -1966,6 +1980,7 @@ py_library( ":tensor_array_ops", ":util", ":variable_scope", + "//tensorflow/python/eager:context", ], ) @@ -2284,7 +2299,10 @@ py_library( srcs_version = "PY2AND3", deps = [ ":array_ops", + ":constant_op", ":data_flow_ops_gen", + ":dtypes", + ":errors", ":framework_ops", ":math_ops", ":tensor_shape", @@ -2626,6 +2644,7 @@ py_library( ":init_ops", ":io_ops", ":io_ops_gen", + ":layers_base", ":lib", ":lookup_ops", ":math_ops", @@ -2644,6 +2663,7 @@ py_library( ":util", ":variable_scope", ":variables", + "//tensorflow/python/eager:backprop", "//third_party/py/numpy", "@six_archive//:six", ], @@ -2981,10 +3001,12 @@ tf_py_wrap_cc( "util/stat_summarizer.i", "util/tfprof.i", "util/transform_graph.i", + "util/util.i", ], deps = [ ":cost_analyzer_lib", ":model_analyzer_lib", + ":cpp_python_util", ":cpp_shape_inference", ":kernel_registry", ":numpy_lib", @@ -3197,7 +3219,10 @@ cuda_py_test( ":variables", "//third_party/py/numpy", ], - tags = ["oss_serial"], + tags = [ + "no_oss", # Test flaky due to port collisions. + "oss_serial", + ], ) tf_py_test( @@ -3213,6 +3238,7 @@ tf_py_test( ":variables", ], tags = [ + "no_oss", # Test flaky due to port collisions. "notsan", # data race due to b/62910646 "oss_serial", ], @@ -3396,7 +3422,6 @@ cuda_py_test( ":training", ":platform_test", ":client_testlib", - ":variable_scope", "//third_party/py/numpy", ], ) @@ -3681,7 +3706,10 @@ py_test( size = "medium", srcs = ["training/monitored_session_test.py"], srcs_version = "PY2AND3", - tags = ["no_windows"], + tags = [ + "no_windows", + "notsan", # b/67945581 + ], deps = [ ":array_ops", ":client_testlib", @@ -3764,6 +3792,7 @@ py_library( ":summary_op_util", ":summary_ops", ":util", + "//tensorflow/python/eager:context", "//third_party/py/numpy", "@six_archive//:six", ], @@ -4007,6 +4036,12 @@ filegroup( visibility = ["//tensorflow:__subpackages__"], ) +filegroup( + name = "hidden_ops", + srcs = ["ops/hidden_ops.txt"], + visibility = ["//tensorflow:__subpackages__"], +) + cuda_py_test( name = "accumulate_n_benchmark", size = "large", @@ -4179,6 +4214,19 @@ cuda_py_test( main = "client/session_benchmark.py", ) +cuda_py_test( + name = "nn_grad_test", + size = "small", + srcs = ["ops/nn_grad_test.py"], + additional_deps = [ + ":client_testlib", + ":framework_for_generated_wrappers", + ":nn_grad", + ":nn_ops", + "//third_party/py/numpy", + ], +) + py_library( name = "tf_item", srcs = [ diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py index 8d9c5de9adaa20bcaef1df181d9480a58dc1eec4..af34aca3e345ff6d12f471f289b77001b40c00bf 100644 --- a/tensorflow/python/__init__.py +++ b/tensorflow/python/__init__.py @@ -262,6 +262,7 @@ _allowed_symbols.extend([ 'VERSION', 'GIT_VERSION', 'COMPILER_VERSION', + 'CXX11_ABI_FLAG', ]) # Remove all extra symbols that don't have a docstring or are not explicitly @@ -280,6 +281,7 @@ _exported_dunders = set([ '__version__', '__git_version__', '__compiler_version__', + '__cxx11_abi_flag__', ]) # Expose symbols minus dunders, unless they are whitelisted above. diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i index 4200439dc6bcc60b2a5d0feac41f54534fad5774..40731aba7d4ed8bb281191d719b3ddfcd2db1ddc 100644 --- a/tensorflow/python/client/tf_session.i +++ b/tensorflow/python/client/tf_session.i @@ -45,6 +45,9 @@ tensorflow::ImportNumpy(); // Compiler %constant const char* __compiler_version__ = tf_compiler_version(); +// _GLIBCXX_USE_CXX11_ABI flag value +%constant const int __cxx11_abi_flag__ = tf_cxx11_abi_flag(); + // Release the Python GIL for the duration of most methods. %exception { Py_BEGIN_ALLOW_THREADS; diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 9ea6a2cf8e38a96a51c4dd42a799be20b05da20d..343f316281b862c8523ec2cf0375a5ba9e9520ca 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -25,6 +25,7 @@ import numpy as np from tensorflow.python.data.ops import iterator_ops from tensorflow.python.data.util import nest +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import function @@ -80,7 +81,14 @@ class Dataset(object): Returns: An `Iterator` over the elements of this dataset. + + Raises: + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "dataset.make_initializable_iterator is not supported when eager " + "execution is enabled.") if shared_name is None: shared_name = "" iterator_resource = gen_dataset_ops.iterator( @@ -102,7 +110,14 @@ class Dataset(object): Returns: An `Iterator` over the elements of this dataset. + + Raises: + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "dataset.make_one_shot_iterator is not supported when eager " + "execution is enabled.") # NOTE(mrry): We capture by value here to ensure that `_make_dataset()` is # a 0-argument function. @function.Defun(capture_by_value=True) @@ -201,7 +216,10 @@ class Dataset(object): with self._lock: ret = self._next_id self._next_id += 1 - return ret + # NOTE(mrry): Explicitly create an array of `np.int64` because implicit + # casting in `py_func()` will create an array of `np.int32` on Windows, + # leading to a runtime error. + return np.array(ret, dtype=np.int64) def get_iterator(self, iterator_id): return self._iterators[iterator_id] @@ -1054,21 +1072,21 @@ class RangeDataset(Dataset): def _parse_args(self, *args): if len(args) == 1: self._start = self._build_tensor(0, "start") - self._stop = args[0] + self._stop = self._build_tensor(args[0], "stop") self._step = self._build_tensor(1, "step") elif len(args) == 2: - self._start = args[0] - self._stop = args[1] + self._start = self._build_tensor(args[0], "start") + self._stop = self._build_tensor(args[1], "stop") self._step = self._build_tensor(1, "step") elif len(args) == 3: - self._start = args[0] - self._stop = args[1] - self._step = args[2] + self._start = self._build_tensor(args[0], "start") + self._stop = self._build_tensor(args[1], "stop") + self._step = self._build_tensor(args[2], "step") else: raise ValueError("Invalid arguments to RangeDataset: %s" % str(args)) def _build_tensor(self, int64_value, name): - return constant_op.constant(int64_value, dtype=dtypes.int64, name=name) + return ops.convert_to_tensor(int64_value, dtype=dtypes.int64, name=name) def _as_variant_tensor(self): return gen_dataset_ops.range_dataset( @@ -1214,7 +1232,8 @@ class BatchDataset(Dataset): """See `Dataset.batch()` for details.""" super(BatchDataset, self).__init__() self._input_dataset = input_dataset - self._batch_size = batch_size + self._batch_size = ops.convert_to_tensor(batch_size, dtype=dtypes.int64, + name="batch_size") def _as_variant_tensor(self): return gen_dataset_ops.batch_dataset( @@ -1282,7 +1301,8 @@ class PaddedBatchDataset(Dataset): """See `Dataset.batch()` for details.""" super(PaddedBatchDataset, self).__init__() self._input_dataset = input_dataset - self._batch_size = batch_size + self._batch_size = ops.convert_to_tensor(batch_size, dtype=dtypes.int64, + name="batch_size") padding_values = (padding_values if padding_values is not None else self._default_padding(input_dataset)) self._padded_shapes = nest.map_structure_up_to( @@ -1506,8 +1526,10 @@ class InterleaveDataset(Dataset): self._map_func = tf_map_func self._map_func.add_to_graph(ops.get_default_graph()) - self._cycle_length = ops.convert_to_tensor(cycle_length, dtype=dtypes.int64) - self._block_length = ops.convert_to_tensor(block_length, dtype=dtypes.int64) + self._cycle_length = ops.convert_to_tensor(cycle_length, dtype=dtypes.int64, + name="cycle_length") + self._block_length = ops.convert_to_tensor(block_length, dtype=dtypes.int64, + name="block_length") def _as_variant_tensor(self): return gen_dataset_ops.interleave_dataset( @@ -1584,7 +1606,8 @@ class PrefetchDataset(Dataset): """See `Dataset.prefetch()` for details.""" super(PrefetchDataset, self).__init__() self._input_dataset = input_dataset - self._buffer_size = ops.convert_to_tensor(buffer_size, dtype=dtypes.int64) + self._buffer_size = ops.convert_to_tensor(buffer_size, dtype=dtypes.int64, + name="buffer_size") def _as_variant_tensor(self): return gen_dataset_ops.prefetch_dataset( diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD index b68b6e05b674931be0115e768818b263fc7bf202..68b97ddbe3048b7aef18fcf8cc2b41ee545ee55f 100644 --- a/tensorflow/python/debug/BUILD +++ b/tensorflow/python/debug/BUILD @@ -897,8 +897,8 @@ cuda_py_test( "//tensorflow/python:variables", ], tags = [ + "no_oss", # Test flaky due to port collisions. "no_windows", - "nomac", # TODO(cais): Install of futures and grpcio on all macs. "notsan", "oss_serial", ], diff --git a/tensorflow/python/debug/wrappers/framework.py b/tensorflow/python/debug/wrappers/framework.py index 1947d749735929f3ae9a23455ad618a3fa3bbcce..4e243cb6c9649a24009a0c9ac501c59eaac3bd79 100644 --- a/tensorflow/python/debug/wrappers/framework.py +++ b/tensorflow/python/debug/wrappers/framework.py @@ -551,6 +551,10 @@ class BaseDebugWrapperSession(session.SessionInterface): return (self._thread_name_filter_pattern and not self._thread_name_filter_pattern.match(thread_name)) + def run_step_fn(self, step_fn): + return step_fn( + monitored_session.MonitoredSession.StepContext(self._sess, self.run)) + def partial_run_setup(self, fetches, feeds=None): """Sets up the feeds and fetches for partial runs in the session.""" raise NotImplementedError( @@ -792,7 +796,7 @@ class NonInteractiveDebugWrapperSession(BaseDebugWrapperSession): def __init__(self, sess, watch_fn=None, thread_name_filter=None, pass_through_operrors=False): - """Constructor of DumpingDebugWrapperSession. + """Constructor of NonInteractiveDebugWrapperSession. Args: sess: The TensorFlow `Session` object being wrapped. diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 69b96df87c16c86ecc48a679f3616f568efa58ab..f5b946ec263c40bd62261297ef55ffa52cb2c042 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -93,7 +93,6 @@ cuda_py_test( ":test", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", - "//third_party/py/numpy", ], ) @@ -123,10 +122,12 @@ cuda_py_test( ":core", ":execute", ":test", + "//third_party/py/numpy", "//tensorflow/python:dtypes", "//tensorflow/python:errors", - "//tensorflow/python:pywrap_tensorflow", + "//tensorflow/python:framework_ops", "//tensorflow/python:framework_test_lib", + "//tensorflow/python:pywrap_tensorflow", ], ) @@ -150,9 +151,9 @@ py_library( srcs_version = "PY2AND3", visibility = ["//tensorflow:internal"], deps = [ - ":context", ":core", "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_ops", ], ) @@ -390,14 +391,14 @@ py_test( ], ) -py_test( +cuda_py_test( name = "ops_test", srcs = ["ops_test.py"], - srcs_version = "PY2AND3", - deps = [ + additional_deps = [ ":context", ":execute", ":test", + "//third_party/py/numpy", "//tensorflow/python:array_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:dtypes", @@ -408,7 +409,7 @@ py_test( "//tensorflow/python:math_ops", "//tensorflow/python:random_ops", "//tensorflow/python:sparse_ops", - "//third_party/py/numpy", + "//tensorflow/python:tensor_shape", ], ) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 0060dd0c1c797d7f13d1b0b01bd339186a861b59..6f7f2117be6ffaa426b1ff59c18140c0cc9e552a 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -35,7 +35,9 @@ from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.util import nest from tensorflow.python.util import tf_inspect @@ -321,12 +323,18 @@ def implicit_val_and_grad(f): ``` Args: - f: The function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. Returns: A function which, when called, returns a tuple pair. Its first element is the value to which the function evaluates. Its second element is list of (gradient, variable) pairs. + + Raises: + ValueError: if `f` returns None. """ # TODO(cais): Remove calls to tf.constant() once the gradients functions # accept lists and np.ndarrays. @@ -334,11 +342,22 @@ def implicit_val_and_grad(f): def grad_fn(*args): """Computes the gradient of the wrapped function.""" tape.push_new_tape() - end_node = f(*args) - variables = tape.top_tape_watched_variables() + try: + end_node = f(*args) + if end_node is None: + raise ValueError("Cannot differentiate a function that returns None; " + "did you forget to return a value from {}?".format( + f.__name__)) + variables = tape.top_tape_watched_variables() + finally: + popped_tape = tape.pop_tape() sources = [x.handle for x in variables] + + if not sources: + raise ValueError("No trainable variables were accessed while the " + "function was being computed.") grad = imperative_grad.imperative_grad(_default_vspace, - tape.pop_tape(), + popped_tape, nest.flatten(end_node), sources) return end_node, list(zip(grad, variables)) @@ -376,7 +395,10 @@ def implicit_grad(f): ``` Args: - f: The function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. Returns: A function which, when called, returns a list of (gradient, variable) pairs. @@ -391,12 +413,22 @@ def implicit_grad(f): return grad_fn -def _get_arg_spec(f, params): - args = tf_inspect.getargspec(f).args +def _get_arg_spec(f, params, param_args): + """The positions of the parameters of f to be differentiated in param_args.""" + try: + args = tf_inspect.getargspec(f).args + except TypeError as e: + # TypeError can happen when f is a callable object. + if params is None: + return range(len(param_args)) + elif all(isinstance(x, int) for x in params): + return params + raise ValueError("Either callable provided is not a function or could not " + "inspect its arguments by name: %s. Original error: %s" + % (f, e)) if params is None: if not args: - raise ValueError("When params is None the differentiated function cannot" - " only take arguments by *args and **kwds.") + return range(len(param_args)) return range(len(args)) elif all(isinstance(x, six.string_types) for x in params): return [args.index(n) for n in params] @@ -449,7 +481,12 @@ def gradients_function(f, params=None): ``` Args: - f: function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. If desired, the tensors can be elementwise multiplied by the + tensors passed as the `dy` keyword argument to the returned gradient + function. params: list of parameter names of f or list of integers indexing the parameters with respect to which we'll differentiate. Passing None differentiates with respect to all parameters. @@ -541,7 +578,12 @@ def val_and_grad_function(f, params=None): ``` Args: - f: function to be differentiated. + f: function to be differentiated. If `f` returns a scalar, this scalar will + be differentiated. If `f` returns a tensor or list of tensors, by default + a scalar will be computed by adding all their values to produce a single + scalar. If desired, the tensors can be elementwise multiplied by the + tensors passed as the `dy` keyword argument to the returned gradient + function. params: list of parameter names of f or list of integers indexing the parameters with respect to which we'll differentiate. Passing `None` differentiates with respect to all parameters. @@ -555,28 +597,83 @@ def val_and_grad_function(f, params=None): ValueError: if the params are not all strings or all integers. """ - parameter_positions = _get_arg_spec(f, params) - def decorated(*args, **kwds): """Computes the value and gradient of the decorated function.""" dy = kwds.pop("dy", None) - if dy is not None: - dy = ops.convert_to_tensor(dy) + if kwds: + raise ValueError("Functions to be differentiated cannot " + "receive keyword arguments.") + val, vjp = make_vjp(f, params)(*args, **kwds) + return val, vjp(dy=dy) + + return decorated + + +def make_vjp(f, params=None): + """Returns a function that computes f and is vjp w.r.t. params. + + The term "vjp" here is an abbreviation for vector-jacobian product. + + Args: + f: the function to be differentiated. + params: the parameters (numbers or names) to differentiate with respect to. + A value of None will differentiate with respect to all parameters. + + Returns: + A function, which when called, returns a tuple (value, vjp), where: + - value is the result of calling f. + - vjp is a function, which takes a vector as an argument and + returns the product of that vector with the Jacobian of f. + Providing no argument to vjp is equivalent to providing a + vector of ones. + + For example, + ```python + def f(x): + return x * x + + wrapped_fn = tfe.make_vjp(f) + result, vjp = wrapped_fn(tf.constant(3.0)) + # result is 9.0 + vjp() # the vjp function rturns 6.0 + + Raises: + ValueError: if `f` returns None. + """ + + def decorated(*args, **kwds): + """Computes the value and gradient of the decorated function.""" + parameter_positions = _get_arg_spec(f, params, args) assert not kwds, "The gradient function can't take keyword arguments." tape.push_new_tape() - sources = [] - args = [ - ops.convert_to_tensor(args[i]) if i in parameter_positions else args[i] - for i in range(len(args)) - ] - args = _ensure_unique_tensor_objects(parameter_positions, args) - for i in parameter_positions: - sources.append(args[i]) - tape.watch(args[i]) - result = f(*args) - return result, imperative_grad.imperative_grad( - _default_vspace, tape.pop_tape(), nest.flatten(result), sources, - output_gradients=nest.flatten(dy) if dy is not None else None) + try: + sources = [] + args = [ + ops.convert_to_tensor(args[i]) + if i in parameter_positions else args[i] + for i in range(len(args)) + ] + args = _ensure_unique_tensor_objects(parameter_positions, args) + for i in parameter_positions: + sources.append(args[i]) + tape.watch(args[i]) + result = f(*args) + if result is None: + raise ValueError("Cannot differentiate a function that returns None; " + "did you forget to return a value from {}?".format( + f.__name__)) + flat_result = nest.flatten(result) + flat_result = [gen_array_ops.identity(x) for x in flat_result] + result = nest.pack_sequence_as(result, flat_result) + finally: + t = tape.pop_tape() + def vjp(dy=None): + if dy is not None: + dy = [ops.convert_to_tensor(x) for x in nest.flatten(dy)] + return imperative_grad.imperative_grad( + _default_vspace, t, nest.flatten(result), sources, + output_gradients=dy) + return result, vjp return decorated @@ -621,49 +718,108 @@ def _aggregate_grads(gradients): return ops.IndexedSlices(values, indices, dense_shape) -# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total -# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation -# so as to release the gradient tensor to save memory. -_MIN_AGGREGATE_COUNT = 4 -_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024 +def _num_elements(grad): + """The number of elements in the `grad` tensor.""" + if isinstance(grad, ops.Tensor): + return functools.reduce(operator.mul, grad._shape_tuple(), 1) # pylint: disable=protected-access + if isinstance(grad, ops.IndexedSlices): + return functools.reduce(operator.mul, grad.values._shape_tuple(), 1) # pylint: disable=protected-access + raise ValueError("`grad` not a Tensor or IndexedSlices.") -def _add_new_grads(gradients, gradients_size, tid, grad): - """Adds a new gradient and maybe aggregate the gradients. +_default_vspace = imperative_grad.VSpace( + num_elements_fn=_num_elements, + aggregate_fn=_aggregate_grads, + tensor_id=ops.tensor_id, + zeros=array_ops.zeros, + ones_like=lambda x: ops.convert_to_tensor(array_ops.ones_like(x))) - Args: - gradients: A dict map from tensor id to list of gradients. - gradients_size: A dict map from tensor id to its total units. Might - not be initialized. - tid: Tensor id. - grad: New gradient for the `tid`, either a Tensor or IndexedSlices. - Raises: - ValueError: if `grad` is neight Tensor nor IndexedSlices. - """ - tensor_grads = gradients[tid] - tensor_grads.append(grad) - if len(tensor_grads) < _MIN_AGGREGATE_COUNT: - return - elif tid not in gradients_size: - if isinstance(grad, ops.Tensor): - size = functools.reduce(operator.mul, grad._shape_tuple(), 1) # pylint: disable=protected-access - elif isinstance(grad, ops.IndexedSlices): - size = functools.reduce(operator.mul, grad.values._shape_tuple(), 1) # pylint: disable=protected-access - else: - raise ValueError("Unexpected gradient type: %s" % type(grad)) - gradients_size[tid] = size - else: - size = gradients_size[tid] +class GradientTape(object): + """Records operations to use to compute gradients. - # For simplicity, assume each element to be 4 bytes now. - if len(tensor_grads) * size * 4 > _MIN_AGGREGATE_BYTES: - gradients[tid] = [_aggregate_grads(tensor_grads)] + Operations are recorded if: + - they happen in code marked by this context manager + - at least one of their inputs is being watched + Outputs of recorded operations are watched. Variables are automatically + watched and tensors can be manually watched by calling the watch method on the + context manager. -_default_vspace = imperative_grad.VSpace( - add_new_grads_fn=_add_new_grads, - aggregate_fn=_aggregate_grads, - tensor_id=ops.tensor_id, - zeros=array_ops.zeros, - ones_like=array_ops.ones_like) + Example usage: + + ```python + with tfe.GradientTape() as g: + x = tf.constant(3.0) + g.watch(x) + y = x * x + grad = g.gradient(y, [x])[0] + assert grad.numpy() == 6.0 + ``` + + It is possible to use GradientTapes to compute higher-order derivatives as + follows: + + ```python + with tfe.GradientTape() as g: + x = tf.constant(3.0) + g.watch(x) + y = x * x + with tfe.GradientTape() as gg: + gg.watch(y) + z = 2 * y + inner_grad = gg.gradient(z, [y])[0] + assert inner_grad.numpy() == 2 + y = y + inner_grad + grad = g.gradient(y, [x])[0] + assert grad.numpy() == 6.0 + ``` + """ + + def __init__(self): + self._tape = None + + def __enter__(self): + tape.push_new_tape() + return self + + def __exit__(self, typ, value, traceback): + self._tape = tape.pop_tape() + + def watch(self, tensor): + """Ensures that `tensor` is being traced by this tape. + + Args: + tensor: a Tensor or Variable a list of Tensors or Variables. + """ + for t in nest.flatten(tensor): + if isinstance(t, resource_variable_ops.ResourceVariable): + t = t.handle + tape.watch(t) + + def gradient(self, target, sources): + """Computes the gradient using information traced by the tape. + + Args: + target: the tensor to be differentiated. + sources: a list of Tensors or Variables, the target will be + differentiated with respect to the sources. + + Returns: + a list of Tensors (or IndexedSlices, or None), one for each element in + `sources`. + + Raises: + RuntimeError: if called inside the context of the tape, or if called more + than once. + """ + if self._tape is None: + raise RuntimeError("GradientTape.gradient can only be called once, and " + "only when the context manager has exited.") + sources = [x.handle if isinstance(x, resource_variable_ops.ResourceVariable) + else x + for x in sources] + grad = imperative_grad.imperative_grad( + _default_vspace, self._tape, [target], sources) + self.tape = None + return grad diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 2409a7b19856bebba507c2d08d301d838238233d..ed54b8e12e74d2187cef6383fa77c7a8280c6d73 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -16,12 +16,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + import numpy as np from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.eager import custom_gradient +from tensorflow.python.eager import imperative_grad from tensorflow.python.eager import tape from tensorflow.python.eager import test from tensorflow.python.framework import constant_op @@ -89,8 +92,8 @@ class BackpropTest(test.TestCase): return math_ops.add(c, constant_op.constant(3.0)) grads_and_vars = backprop.implicit_grad(fn)() - self.assertEqual(grads_and_vars[0][0].numpy(), 1.0) - self.assertEqual(id(grads_and_vars[0][1]), id(x)) + self.assertAllEqual(grads_and_vars[0][0], 1.0) + self.assertAllEqual(id(grads_and_vars[0][1]), id(x)) def testDy(self): @@ -98,7 +101,7 @@ class BackpropTest(test.TestCase): return x grad_fn = backprop.gradients_function(f) - self.assertAllEqual(2., grad_fn(1., dy=2.)[0].numpy()) + self.assertAllEqual(2., grad_fn(1., dy=2.)[0]) def testImplicitGradOverEmbeddingLookup(self): batch_size = 8 @@ -130,13 +133,13 @@ class BackpropTest(test.TestCase): tf_opt = training.GradientDescentOptimizer(0.1) tf_embedding.initializer.run() - self.assertAllClose(tf_grad.indices.eval(), grad.indices.numpy()) - self.assertAllClose(tf_grad.values.eval(), grad.values.numpy()) + self.assertAllClose(tf_grad.indices.eval(), grad.indices) + self.assertAllClose(tf_grad.values.eval(), grad.values) tf_opt.apply_gradients([(tf_grad, tf_embedding)]).run() expected = tf_embedding.eval() opt.apply_gradients([(grad, embedding)]) - self.assertAllClose(expected, embedding.read_value().numpy()) + self.assertAllClose(expected, embedding.read_value()) def testGradientNone(self): @@ -166,7 +169,17 @@ class BackpropTest(test.TestCase): f = constant_op.constant([[0.1]]) grad = backprop.gradients_function(second, [0])(f)[0] - self.assertAllEqual([[0.0]], grad.numpy()) + self.assertAllEqual([[0.0]], grad) + + def testMakeVJP(self): + + def f(x): + return x * x + + wrapped_fn = backprop.make_vjp(f) + result, vjp = wrapped_fn(constant_op.constant(3.0)) + self.assertAllEqual(result, 9.0) + self.assertAllEqual(vjp(2.0)[0], 12.0) def testGradGrad(self): @@ -179,7 +192,7 @@ class BackpropTest(test.TestCase): gradgrad = backprop.gradients_function(grad, [0]) - self.assertAllEqual(gradgrad(constant_op.constant(3.0))[0].numpy(), 2.0) + self.assertAllEqual(gradgrad(constant_op.constant(3.0))[0], 2.0) def testGradGradExp(self): @@ -189,7 +202,7 @@ class BackpropTest(test.TestCase): gradgrad = backprop.gradients_function(grad, [0]) - self.assertAllEqual(gradgrad(constant_op.constant(0.0))[0].numpy(), 1.0) + self.assertAllEqual(gradgrad(constant_op.constant(0.0))[0], 1.0) def testGPU(self): if not context.context().num_gpus(): @@ -198,13 +211,13 @@ class BackpropTest(test.TestCase): def fn(x): with context.device('/gpu:0'): b = constant_op.constant(2.0) - c = math_ops.add(x.as_gpu_tensor(), b) - # TODO(apassos): remove as_cpu_tensor below by making TensorVSPace aware + c = math_ops.add(x.gpu(), b) + # TODO(apassos): remove cpu below by making TensorVSPace aware # of devices. - return math_ops.add(c, constant_op.constant(3.0)).as_cpu_tensor() + return math_ops.add(c, constant_op.constant(3.0)).cpu() grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testGPUImplicitGrad(self): if not context.context().num_gpus(): @@ -219,7 +232,7 @@ class BackpropTest(test.TestCase): return v.read_value() self.assertEqual( - backprop.implicit_grad(f)()[0][0].as_cpu_tensor().numpy(), 1.0) + backprop.implicit_grad(f)()[0][0].cpu().numpy(), 1.0) def testCPU(self): @@ -229,21 +242,21 @@ class BackpropTest(test.TestCase): return math_ops.add(c, constant_op.constant(3.0)) grad = backprop.gradients_function(fn, [0])(constant_op.constant(1.0))[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testTensorCopyGPU2CPU2GPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') def f(a, b): - return a.as_cpu_tensor() + b.as_cpu_tensor() + return a.cpu() + b.cpu() with context.device('/gpu:0'): a = constant_op.constant(1.0) b = constant_op.constant(2.0) grad = backprop.gradients_function(f, [0])(a, b)[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testEmptyParams(self): @@ -253,8 +266,8 @@ class BackpropTest(test.TestCase): x = constant_op.constant(1.0) y = constant_op.constant(2.0) dx, dy = backprop.gradients_function(fn)(x, y) - self.assertAllEqual(dx.numpy(), y.numpy()) - self.assertAllEqual(dy.numpy(), x.numpy()) + self.assertAllEqual(dx, y.numpy()) + self.assertAllEqual(dy, x.numpy()) def testUnconnectedNone(self): v = resource_variable_ops.ResourceVariable( @@ -266,6 +279,27 @@ class BackpropTest(test.TestCase): self.assertEqual(backprop.implicit_grad(f)()[0][0], None) + def testGradientTape(self): + with backprop.GradientTape() as g: + x = constant_op.constant(3.0) + g.watch(x) + y = x * x + with backprop.GradientTape() as gg: + gg.watch(y) + z = 2 * y + inner_grad = gg.gradient(z, [y])[0] + self.assertEqual(inner_grad.numpy(), 2.0) + y += inner_grad + grad = g.gradient(y, [x])[0] + self.assertEqual(grad.numpy(), 6.0) + + def testGradientTapeVariable(self): + v = resource_variable_ops.ResourceVariable(1.0, name='v') + with backprop.GradientTape() as g: + y = v * v + grad = g.gradient(y, [v])[0] + self.assertAllEqual(grad, 2.0) + def testEmptyParamsForValueAndGradFunction(self): def fn(a, b): return a * b @@ -274,9 +308,9 @@ class BackpropTest(test.TestCase): x = 2.0 y = 3.0 val, (dx, dy) = val_and_grads_fn(x, y) - self.assertAllClose(val.numpy(), x * y) - self.assertAllEqual(dx.numpy(), y) - self.assertAllEqual(dy.numpy(), x) + self.assertAllClose(val, x * y) + self.assertAllEqual(dx, y) + self.assertAllEqual(dy, x) def testNonEmptyParamsForValueAndGradFunction(self): def fn(a, b): @@ -286,9 +320,9 @@ class BackpropTest(test.TestCase): x = 2.0 y = 3.0 val, grads = val_and_grad_fn(x, y) - self.assertAllClose(val.numpy(), x * y) + self.assertAllClose(val, x * y) self.assertEqual(1, len(grads)) - self.assertAllEqual(grads[0].numpy(), x) + self.assertAllEqual(grads[0], x) def testTensorCopyCPU2GPU2CPU(self): if not context.context().num_gpus(): @@ -298,15 +332,15 @@ class BackpropTest(test.TestCase): # back: e (cpu) -> add (cpu) -> c (cpu->gpu) -> add (gpu) -> grad (gpu->cpu) def f(a, b): with context.device('/gpu:0'): - c = math_ops.add(a.as_gpu_tensor(0), b.as_gpu_tensor(0)) - return math_ops.add(c.as_cpu_tensor(), constant_op.constant(3.0)) + c = math_ops.add(a.gpu(0), b.gpu(0)) + return math_ops.add(c.cpu(), constant_op.constant(3.0)) with context.device('/cpu:0'): a = constant_op.constant(1.0) b = constant_op.constant(2.0) grad = backprop.gradients_function(f, [0])(a, b)[0] - self.assertEqual(grad.numpy(), 1.0) + self.assertAllEqual(grad, 1.0) def testGetAttrType(self): typ = backprop.op_attr_type('Add', 'T') @@ -349,6 +383,46 @@ class BackpropTest(test.TestCase): [tensor_shape.TensorShape(s).as_proto() for s in shape_list], backprop.make_attr([pywrap_tensorflow.TF_ATTR_SHAPE], shape_list)) + def testArgsGradientFunction(self): + + def f(*args): + return args[0] * args[0] + + grad = backprop.gradients_function(f) + self.assertAllEqual(grad(1.0)[0], 2.0) + + def testPartial(self): + + def f(x, y): + return x * y + + part = functools.partial(f, constant_op.constant(2.0)) + self.assertAllEqual( + backprop.gradients_function(part)(constant_op.constant(1.0))[0], + 2.0) + + def testReturnSameThing(self): + + def f(x): + return x, 2 * x + + self.assertAllEqual(backprop.gradients_function(f)(1.0)[0], 3.0) + + def testExceptionSafety(self): + + def f(unused_x): + raise ValueError() + + try: + backprop.gradients_function(f)(1.0) + except ValueError: + pass + + def real_f(x): + return x * x + + self.assertAllEqual(backprop.gradients_function(real_f)(1.0)[0], 2.0) + def testMultiValueConvertToTensor(self): x = resource_variable_ops.ResourceVariable( initial_value=array_ops.constant([1.0]), name='x') @@ -361,7 +435,7 @@ class BackpropTest(test.TestCase): return math_ops.reduce_mean(b) grad = backprop.implicit_grad(fn)()[0][0] - self.assertAllEqual([1.0], grad.numpy()) + self.assertAllEqual([1.0], grad) def testOutput(self): @@ -371,7 +445,7 @@ class BackpropTest(test.TestCase): x = constant_op.constant([0.0, 1.0, 2.0]) grad = backprop.gradients_function(multiout)(x)[0] - self.assertAllEqual([1.0, 3.0, 5.0], grad.numpy()) + self.assertAllEqual([1.0, 3.0, 5.0], grad) def testMultiValuePreservesIfNotDiffedAgainst(self): @@ -383,7 +457,7 @@ class BackpropTest(test.TestCase): s = [1, 1, 1, 1] grad = backprop.gradients_function(tfe_conv2d, params=(0,))(i, k, s)[0] - self.assertAllEqual([[[[2.0]]]], grad.numpy()) + self.assertAllEqual([[[[2.0]]]], grad) def testSameObjectForMultipleArguments(self): @@ -417,7 +491,8 @@ class BackpropTest(test.TestCase): add_n.append(1) context.context().add_post_execution_callback(callback) - v = resource_variable_ops.ResourceVariable(constant_op.constant(2.0)) + v = resource_variable_ops.ResourceVariable(constant_op.constant(2.0), + name='v') def fn(): outputs = [] for _ in range(20): @@ -432,21 +507,21 @@ class BackpropTest(test.TestCase): # Reduce the aggregation limit, cause the backprop to do some # early aggregation. # pylint: disable=protected-access - old_cnt = backprop._MIN_AGGREGATE_COUNT - old_bytes = backprop._MIN_AGGREGATE_BYTES - backprop._MIN_AGGREGATE_COUNT = 10 - backprop._MIN_AGGREGATE_BYTES = 1 + old_cnt = imperative_grad._MIN_AGGREGATE_COUNT + old_bytes = imperative_grad._MIN_AGGREGATE_BYTES + imperative_grad._MIN_AGGREGATE_COUNT = 10 + imperative_grad._MIN_AGGREGATE_BYTES = 1 _ = backprop.implicit_grad(fn)() self.assertEqual(len(add_n), 6) del add_n[:] # Aggregation is also limited by the memory. - backprop._MIN_AGGREGATE_BYTES = 10000 + imperative_grad._MIN_AGGREGATE_BYTES = 10000 _ = backprop.implicit_grad(fn)() self.assertEqual(len(add_n), 2) - backprop._MIN_AGGREGATE_COUNT = old_cnt - backprop._MIN_AGGREGATE_BYTES = old_bytes + imperative_grad._MIN_AGGREGATE_COUNT = old_cnt + imperative_grad._MIN_AGGREGATE_BYTES = old_bytes # pylint: enable=protected-access context.context().clear_post_execution_callbacks() @@ -472,8 +547,66 @@ class BackpropTest(test.TestCase): grads_and_vars = g() self.assertEqual(1, len(grads_and_vars)) grad, var = grads_and_vars[0] - self.assertEqual(7, grad.numpy()) - self.assertEqual(x, var) + self.assertAllEqual(7, grad) + self.assertAllEqual(x, var) + + def testCustomGradient(self): + + @custom_gradient.custom_gradient + def my_mul(x, y): + result = x*y + + def grad(dr): + return [dr*y, dr*x] + return result, grad + + lr = 0.25 + x = resource_variable_ops.ResourceVariable(2., name='x') + + def loss(x): + return my_mul(2., x.read_value()) + + loss_grads_fn = backprop.implicit_val_and_grad(loss) + + losses = [] + for _ in range(5): + loss, grads_and_vars = loss_grads_fn(x) + losses.append(loss.numpy()) + for (grad, var) in grads_and_vars: + var.assign_sub(lr*grad) + self.assertAllEqual(losses, [4.0, 3., 2., 1., 0.]) + + def testCustomGradientIdentity(self): + + @custom_gradient.custom_gradient + def my_identity(x): + + def grad(dresult): + return [2 * dresult] + + return x, grad + + self.assertAllEqual(backprop.gradients_function(my_identity)(1.0)[0], 2.0) + + def testDifferentiatingFunctionThatReturnsNone(self): + + def fn(x, y): + result = x*y # pylint: disable=unused-variable + + x = constant_op.constant(1) + y = constant_op.constant(2) + + loss_grads_fn = backprop.implicit_val_and_grad(fn) + with self.assertRaisesRegexp( + ValueError, 'Cannot differentiate a function that returns None; ' + 'did you forget to return a value from fn?'): + loss_grads_fn(x, y) + + val_and_grads_fn = backprop.val_and_grad_function(fn) + with self.assertRaisesRegexp( + ValueError, 'Cannot differentiate a function that returns None; ' + 'did you forget to return a value from fn?'): + val_and_grads_fn(x, y) if __name__ == '__main__': diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index 407d1e979c6f95212f5ce47cdfc1f81d1c73104c..ebc9e346c068911bfa1c8d1e8d90ded9267d669c 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -104,7 +104,7 @@ def benchmark_matmul(shape, n, use_gpu=False): transpose_b = (shape[0] != shape[1]) m = random_ops.random_uniform(shape) if use_gpu: - m = m.as_gpu_tensor() + m = m.gpu() # Warm up the GPU - the very first kernel invocation # seems to require a bunch of setup. math_ops.matmul(m, m, transpose_b=transpose_b) @@ -113,7 +113,7 @@ def benchmark_matmul(shape, n, use_gpu=False): return "MatMul {}: {:30s}".format(shape, s) if not use_gpu: - a = m.as_cpu_tensor().numpy() + a = m.cpu().numpy() b = a.T if transpose_b else a with timer(label("np.dot"), iters=n) as iters: for _ in iters: diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 996748a870c16b4537c28ccb00d86e150e4999e1..92f4e15c054bd8cf3886b8c22e414abdfccbdae5 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -26,10 +26,7 @@ import threading from tensorflow.python import pywrap_tensorflow from tensorflow.python.framework import device as pydev from tensorflow.python.framework import errors -from tensorflow.python.platform import app -from tensorflow.python.util import compat from tensorflow.python.util import tf_contextlib -from tensorflow.python.util import tf_inspect GRAPH_MODE = 0 EAGER_MODE = 1 @@ -45,6 +42,10 @@ _device_parsing_cache = {} _MAXINT32 = 2**31 - 1 +DEVICE_PLACEMENT_EXPLICIT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_EXPLICIT +DEVICE_PLACEMENT_WARN = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_WARN +DEVICE_PLACEMENT_SILENT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_SILENT + # TODO(agarwal): better name ? class _EagerContext(threading.local): @@ -57,6 +58,7 @@ class _EagerContext(threading.local): self.mode = _default_mode self.scope_name = "" self.recording_summaries = False + self.summary_writer_resource = None self.scalar_cache = {} @@ -65,22 +67,31 @@ class _EagerContext(threading.local): class Context(object): """Environment in which eager operations execute.""" - def __init__(self, config=None): + def __init__(self, config=None, device_policy=None): """Creates a new Context. Args: config: (Optional.) A `ConfigProto` protocol buffer with configuration - options for the Context. Note that a lot of these options may be - currently unimplemented or irrelevant for EAGER mode. + options for the Context. Note that a lot of these options may be + currently unimplemented or irrelevant when eager execution is enabled. + device_policy: (Optional.) What policy to use when trying to run an + operation on a device with inputs which are not on that device. + Valid values: + tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not + correct. + tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the + right device but raises a warning. + tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might + hide performance problems. """ self._eager_context = _EagerContext() self._context_handle = None self._context_devices = None - self._summary_writer_resource = None self._post_execution_callbacks = [] self._config = config self._seed = None self._initialize_lock = threading.Lock() + self._device_policy = device_policy def _set_global_seed(self, seed): """Set a global eager mode seed for random ops.""" @@ -105,11 +116,19 @@ class Context(object): if self._context_handle is not None: return assert self._context_devices is None - opts = pywrap_tensorflow.TF_NewSessionOptions( - target=compat.as_bytes(""), config=self._config) - with errors.raise_exception_on_not_ok_status() as status: - self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status) - pywrap_tensorflow.TF_DeleteSessionOptions(opts) + opts = pywrap_tensorflow.TFE_NewContextOptions() + try: + with errors.raise_exception_on_not_ok_status() as status: + if self._config is not None: + config_str = self._config.SerializeToString() + pywrap_tensorflow.TFE_ContextOptionsSetConfig( + opts, config_str, len(config_str), status) + if self._device_policy is not None: + pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy( + opts, self._device_policy) + self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status) + finally: + pywrap_tensorflow.TFE_DeleteContextOptions(opts) # Store list of devices self._context_devices = [] with errors.raise_exception_on_not_ok_status() as status: @@ -194,12 +213,12 @@ class Context(object): @property def summary_writer_resource(self): """Returns summary writer resource.""" - return self._summary_writer_resource + return self._eager_context.summary_writer_resource @summary_writer_resource.setter def summary_writer_resource(self, resource): """Sets summary writer resource.""" - self._summary_writer_resource = resource + self._eager_context.summary_writer_resource = resource @property def device_name(self): @@ -423,68 +442,6 @@ def device(name): return context().device(name) -def run(main=None, argv=None): - """Runs the program with an optional main function and argv list. - - The program will run with eager execution enabled. - - Example: - ```python - import tensorflow as tf - # Import subject to future changes: - from tensorflow.contrib.eager.python import tfe - - def main(_): - u = tf.constant(6.0) - v = tf.constant(7.0) - print(u * v) - - if __name__ == "__main__": - tfe.run() - ``` - - Args: - main: the main function to run. - argv: the arguments to pass to it. - """ - enable_eager_execution() - app.run(main, argv) - - -# TODO(apassos): This should not be a part of the public API. -def enable_eager_execution(): - """Enables, for the rest of the lifetime of this program, eager execution. - - If not called immediately on startup risks creating breakage and bugs. Calling - this method more than once in the same process will lead to an exception. - - Example: - ```python - # Before eager execution is enabled, `Tensor`s are symbolic and do not hold - # concrete values (they are to be executed in a `tf.Session`). - assert not hasattr(tf.multiply(6, 7), "numpy") - - tfe.enable_eager_execution() - - # After eager execution is enabled, operations are executed as they are - # defined and `Tensor`s hold concrete values, which can be accessed as - # `numpy.ndarray`s through the `numpy()` method. - assert tf.multiply(6, 7).numpy() == 42 - ``` - - Raises: - ValueError: If this method has already been invoked in the current process. - """ - global _default_mode - if _default_mode == EAGER_MODE: - func_name = ( - "tfe." + tf_inspect.getframeinfo(tf_inspect.currentframe()).function) - raise ValueError( - "Do not call %s more than once in the same process. Note eager-mode " - "methods such as tfe.run() also call %s." % (func_name, func_name)) - _default_mode = EAGER_MODE - - def list_devices(): """List the names of the available devices. diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py index 041d388fad11704e8f821af642de65198bb4fc45..2449162dcaa47cb71dde3be70675654709fec794 100644 --- a/tensorflow/python/eager/core_test.py +++ b/tensorflow/python/eager/core_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import threading +import numpy as np + from tensorflow.core.protobuf import config_pb2 from tensorflow.python import pywrap_tensorflow from tensorflow.python.eager import context @@ -29,6 +31,7 @@ from tensorflow.python.eager import test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -119,13 +122,13 @@ class TFETest(test_util.TensorFlowTestCase): if not context.context().num_gpus(): self.skipTest('No GPUs found') - x = constant_op.constant(1.).as_gpu_tensor() + x = constant_op.constant(1.).gpu() with context.device('gpu:0'): y = constant_op.constant(2.) # Add would fail if t2 were not on GPU result = execute( b'Add', 1, inputs=[x, y], - attrs=('T', x.dtype.as_datatype_enum))[0].as_cpu_tensor().numpy() + attrs=('T', x.dtype.as_datatype_enum))[0].cpu().numpy() self.assertEqual(3, result) def testCopyBetweenDevices(self): @@ -133,29 +136,29 @@ class TFETest(test_util.TensorFlowTestCase): self.skipTest('No GPUs found') x = constant_op.constant([[1., 2.], [3., 4.]]) - x = x.as_cpu_tensor() - x = x.as_gpu_tensor() - x = x.as_gpu_tensor() - x = x.as_cpu_tensor() + x = x.cpu() + x = x.gpu() + x = x.gpu() + x = x.cpu() # Invalid device with self.assertRaises(RuntimeError): - x.as_gpu_tensor(context.context().num_gpus() + 1) + x.gpu(context.context().num_gpus() + 1) def testNumpyForceCPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') cpu = constant_op.constant([[1., 2.], [3., 4.]]) - c2g = cpu.as_gpu_tensor() - self.assertAllEqual(c2g.numpy(), cpu.numpy()) + c2g = cpu.gpu() + self.assertAllEqual(c2g, cpu.numpy()) def testCopyFromCPUToCPU(self): ta = constant_op.constant([[1, 2], [3, 4]]) - tb = ta.as_cpu_tensor() + tb = ta.cpu() self.assertNotEqual(id(ta), id(tb)) - self.assertAllEqual(ta.numpy(), tb.numpy()) + self.assertAllEqual(ta, tb.numpy()) def testRegisterExceptionClass(self): with self.assertRaises(TypeError): @@ -171,7 +174,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[three, five], attrs=('T', three.dtype.as_datatype_enum))[0] - self.assertEqual(15, product.numpy()) + self.assertAllEqual(15, product) def testExecuteTooManyNumOutputs(self): # num_outputs provided is 50, but only one output is produced. @@ -181,20 +184,20 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=50, inputs=[constant_op.constant(3), constant_op.constant(5)], attrs=('T', dtypes.int32.as_datatype_enum))[0] - self.assertEqual(15, product.numpy()) + self.assertAllEqual(15, product) def testMatMulGPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') - three = constant_op.constant([[3.]]).as_gpu_tensor() - five = constant_op.constant([[5.]]).as_gpu_tensor() + three = constant_op.constant([[3.]]).gpu() + five = constant_op.constant([[5.]]).gpu() product = execute( b'MatMul', num_outputs=1, inputs=[three, five], attrs=('transpose_a', False, 'transpose_b', False, 'T', three.dtype.as_datatype_enum))[0] - self.assertEqual([[15.0]], product.numpy()) + self.assertAllEqual([[15.0]], product) def testExecuteStringAttr(self): checked_three = execute( @@ -219,7 +222,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[constant_op.constant(3.0), constant_op.constant(2.9)], attrs=('tolerance', 0.3, 'T', dtypes.float32.as_datatype_enum))[0] - self.assertTrue(almost_equal.numpy()) + self.assertTrue(almost_equal) def testExecuteFloatAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -235,7 +238,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[constant_op.constant(3), constant_op.constant(4)], attrs=('T', dtypes.int32.as_datatype_enum, 'N', 2))[0] - self.assertEqual(7, total.numpy()) + self.assertAllEqual(7, total) def testExecuteIntAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -254,7 +257,7 @@ class TFETest(test_util.TensorFlowTestCase): constant_op.constant([[5]])], attrs=('transpose_a', True, 'transpose_b', False, 'T', dtypes.int32.as_datatype_enum))[0] - self.assertEqual([[15]], product.numpy()) + self.assertAllEqual([[15]], product) def testExecuteShapeAttr(self): execute( @@ -307,7 +310,7 @@ class TFETest(test_util.TensorFlowTestCase): inputs=[constant_op.constant([3.0, 5.0, 7.0])], attrs=('T', dtypes.float32.as_datatype_enum, 'boundaries', [4.0, 6.0]))[0] - self.assertAllEqual([0, 1, 2], b.numpy()) + self.assertAllEqual([0, 1, 2], b) def testExecuteListFloatAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -332,7 +335,7 @@ class TFETest(test_util.TensorFlowTestCase): num_outputs=1, inputs=[constant_op.constant([[[3.0]]])], attrs=('T', dtypes.float32.as_datatype_enum, 'squeeze_dims', [0, 2]))[0] - self.assertAllEqual([3], b.numpy()) + self.assertAllEqual([3], b) def testExecuteListIntAttrBadValue(self): with self.assertRaises(errors.InvalidArgumentError): @@ -404,9 +407,9 @@ class TFETest(test_util.TensorFlowTestCase): inputs=[constant_op.constant(split_dim), constant_op.constant(value)], attrs=('num_split', 3, 'T', dtypes.int32.as_datatype_enum)) - self.assertAllEqual([[0], [3]], x1.numpy()) - self.assertAllEqual([[1], [4]], x2.numpy()) - self.assertAllEqual([[2], [5]], x3.numpy()) + self.assertAllEqual([[0], [3]], x1) + self.assertAllEqual([[1], [4]], x2) + self.assertAllEqual([[2], [5]], x3) def testExecuteBadNumOutputsArgument(self): with self.assertRaises(TypeError): @@ -439,7 +442,7 @@ class TFETest(test_util.TensorFlowTestCase): x = constant_op.constant(1) three_x = add(add(x, x), x) self.assertEquals(dtypes.int32, three_x.dtype) - self.assertEquals(3, three_x.numpy()) + self.assertAllEqual(3, three_x) def testOperationWithNoInputsRunsOnDevice(self): if not context.context().num_gpus(): @@ -447,7 +450,7 @@ class TFETest(test_util.TensorFlowTestCase): shape = constant_op.constant([], dtype=dtypes.int32) # x: Run the "TruncatedNormal" op CPU and copy result to GPU. - x = truncated_normal(shape).as_gpu_tensor() + x = truncated_normal(shape).gpu() # y: Explicitly run the "TruncatedNormal" op on GPU. with context.device('gpu:0'): y = truncated_normal(shape) @@ -460,6 +463,15 @@ class TFETest(test_util.TensorFlowTestCase): with context.device('pu:0'): _ = constant_op.constant(1) + def testConvertMixedEagerTensors(self): + array = np.zeros((), dtype=np.float32) + tensor = constant_op.constant(0., dtype=dtypes.float32) + types, tensors = execute_lib.convert_to_mixed_eager_tensors( + [array, tensor], context.context()) + for typ, t in zip(types, tensors): + self.assertEquals(typ, dtypes.float32) + self.assertIsInstance(t, ops.EagerTensor) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/eager/custom_gradient.py b/tensorflow/python/eager/custom_gradient.py index df116dd8199dd6aed4fb8e88d25ff79a9e568ecf..05460ff9968312528d87f5fc2ad0495b4da2ad1a 100644 --- a/tensorflow/python/eager/custom_gradient.py +++ b/tensorflow/python/eager/custom_gradient.py @@ -22,7 +22,7 @@ from tensorflow.python.eager import context from tensorflow.python.eager import tape from tensorflow.python.framework import ops as tf_ops from tensorflow.python.ops import array_ops -from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import gen_array_ops from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator @@ -69,29 +69,23 @@ def custom_gradient(f): return nest.pack_sequence_as( structure=result, flat_sequence=all_tensors[:len(flat_result)]) - input_tensors = [] - for x in args: - if isinstance(x, tf_ops.Tensor): - input_tensors.append(x) - if isinstance(x, resource_variable_ops.ResourceVariable): - input_tensors.append(x.read_value()) + input_tensors = [tf_ops.convert_to_tensor(x) for x in args] with tape.stop_recording(): result, grad_fn = f(*args, **kwargs) + flat_result = nest.flatten(result) + # TODO(apassos) consider removing the identity below. + flat_result = [gen_array_ops.identity(x) for x in flat_result] - # TODO(apassos): naive uses of custom_gradient will not get the correct - # second derivative this way if they capture any output tensors. Change the - # signature of custom_gradient. def actual_grad_fn(*outputs): return nest.flatten(grad_fn(*outputs)) - flat_result = nest.flatten(result) tape.record_operation( f.__name__, flat_result, input_tensors, actual_grad_fn) flat_result = list(flat_result) - return result + return nest.pack_sequence_as(result, flat_result) return tf_decorator.make_decorator(f, decorated) diff --git a/tensorflow/python/eager/execute.py b/tensorflow/python/eager/execute.py index 04634daba496b80840e24556ad06a29d73157bbe..983c1ea73e59ecdad8def57fc8af36798e2d3c57 100644 --- a/tensorflow/python/eager/execute.py +++ b/tensorflow/python/eager/execute.py @@ -198,8 +198,11 @@ def args_to_matching_eager(l, ctx, default_dtype=None): def convert_to_mixed_eager_tensors(values, ctx): - v = [t if isinstance(t, ops.EagerTensor) else ops.EagerTensor(t, ctx) - for t in values] + v = [ + t if isinstance(t, ops.EagerTensor) else ops.EagerTensor( + t, context=ctx._handle, device=ctx.device_name) # pylint: disable=protected-access + for t in values + ] types = [t.dtype for t in v] return types, v diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index da49517cf9446092a5895af4871a37f9c4d5598e..b1b1de0c41efe351e3972d5c01e8b83fe3c3fccf 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -79,6 +79,22 @@ def capture_tensors(captures): _scoped_captures.tensors = old +def capture_value(tensor_map, value, dtype, name): + """Capture a value from outside the function, to pass in as an extra arg.""" + captured_value = tensor_map.get(ops.tensor_id(value), None) + if captured_value is None: + captured_value = graph_placeholder( + dtype=dtype or value.dtype, shape=value.shape, name=name) + if captured_value.dtype == dtypes.resource: + captured_value._handle_data = value._handle_data # pylint: disable=protected-access + tensor_map[ops.tensor_id(value)] = (value, captured_value) + else: + captured_value = captured_value[1] + tape.record_operation("captured_value", [captured_value], [value], + lambda x: [x]) + return captured_value + + def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False): """Captures a Tensor while building a graph mode function. @@ -100,18 +116,33 @@ def _convert_to_graph_tensor(value, dtype=None, name=None, as_ref=False): if tensor_map is None: # Capturing is not enabled. return constant_op.constant(value.numpy()) - captured_value = tensor_map.get(ops.tensor_id(value), None) - if captured_value is None: - captured_value = graph_placeholder( - dtype=dtype or value.dtype, shape=value.shape, name=name) - if captured_value.dtype == dtypes.resource: - captured_value._handle_data = value._handle_data # pylint: disable=protected-access - tensor_map[ops.tensor_id(value)] = (value, captured_value) - else: - captured_value = captured_value[1] - tape.record_operation("captured_value", [captured_value], [value], - lambda x: [x]) - return captured_value + return capture_value(tensor_map, value, dtype, name) + + +class CapturingGraph(ops.Graph): + + def __init__(self, captures): + super(CapturingGraph, self).__init__() + self._building_function = True + self.captures = captures + + def create_op( + self, + op_type, + inputs, + dtypes, # pylint: disable=redefined-outer-name + input_types=None, + name=None, + attrs=None, + op_def=None, + compute_shapes=True, + compute_device=True): + for i, inp in enumerate(inputs): + if inp.graph is not self: + inputs[i] = capture_value(self.captures, inp, inp.dtype, inp.op.name) + return super(CapturingGraph, self).create_op( + op_type, inputs, dtypes, input_types, name, attrs, op_def, + compute_shapes, compute_device) # TODO(apassos): it'd be really nice if we could scope this registration. @@ -297,7 +328,7 @@ class _GraphModeFunction(object): (args + self._extra_inputs), backward_function) - return self._build_call_outputs(self._returns, real_outputs) + return self._build_call_outputs(real_outputs) def __call__(self, *args): """Executes the passed function in eager mode.""" @@ -316,6 +347,9 @@ class _GraphModeFunction(object): g = ops.get_default_graph() if self._fdef.name not in g._functions: # pylint: disable=protected-access g._add_function(self._fdef) # pylint: disable=protected-access + for f in self._graph._functions.values(): # pylint: disable=protected-access + if f.name not in g._functions: # pylint: disable=protected-access + g._add_function(f) # pylint: disable=protected-access signature = self._fdef.definition.signature args = list(tensor_inputs) + self._extra_inputs op = g.create_op( @@ -325,6 +359,8 @@ class _GraphModeFunction(object): name="FunctionCall", compute_shapes=False) result = op.outputs + if not result: + return op for i, s in enumerate(self._output_shapes): result[i].set_shape(s) else: @@ -335,34 +371,25 @@ class _GraphModeFunction(object): attrs=None, ctx=ctx) - return self._build_call_outputs(self._returns, result) + return self._build_call_outputs(result) - def _build_call_outputs(self, func_outputs, result): + def _build_call_outputs(self, result): """Maps the fdef output list to actual output structure. Args: - func_outputs: The outputs originally defined by the graph function. It - could potentially be a nested structure. result: Output lists defined by FunctionDef. Returns: The actual call output. """ if self._func_outputs is None: return None - if isinstance(self._func_outputs, ops.Tensor): - return result[0] - - outputs = [] - for o in func_outputs: - vo = o - if isinstance(vo, ops.Tensor): - outputs.append(result[self._returns_to_fedf_outputs[id(vo)]]) - elif type(vo) in (tuple, list): - outputs.append(self._build_call_outputs(o, result)) - else: - outputs.append(o) - - return tuple(outputs) if type(func_outputs) is tuple else outputs + outputs_list = nest.flatten(self._func_outputs) + j = 0 + for i, o in enumerate(outputs_list): + if o is not None: + outputs_list[i] = result[j] + j += 1 + return nest.pack_sequence_as(self._func_outputs, outputs_list) def _get_defun_inputs(args): @@ -381,7 +408,8 @@ def _get_defun_inputs(args): def _defun_internal(name, func, args, kwds): """Defines and returns graph-mode version of func.""" with context.graph_mode(): - tmp_graph = ops.Graph() + captures = {} + tmp_graph = CapturingGraph(captures) # Copy the graph collections to ensure summaries and other things work. This # lets the function access (but not mutate) collections of the containing # graph, such as the global step and the summary writer collections. @@ -392,7 +420,6 @@ def _defun_internal(name, func, args, kwds): with tmp_graph.as_default(): func_inputs = _get_defun_inputs(args) - captures = {} with capture_tensors(captures): func_outputs = func(*func_inputs, **kwds) ids = list(sorted(captures.keys())) diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py index 899b6d59b7096ff9717342d26e95b01aa1f2fa1b..243efccac44be1fbba8a00be6683029fc5105a95 100644 --- a/tensorflow/python/eager/function_test.py +++ b/tensorflow/python/eager/function_test.py @@ -31,6 +31,8 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope +from tensorflow.python.ops import variables class FunctionTest(test.TestCase): @@ -52,10 +54,10 @@ class FunctionTest(test.TestCase): t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) out = sq(t) - self.assertAllEqual(out.numpy(), math_ops.matmul(t, t).numpy()) + self.assertAllEqual(out, math_ops.matmul(t, t).numpy()) def testGraphModeWithGradients(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name='v') @function.defun def step(): @@ -65,7 +67,45 @@ class FunctionTest(test.TestCase): return backprop.implicit_grad(inner)()[0][0] - self.assertAllEqual(step().numpy(), 2.0) + self.assertAllEqual(step(), 2.0) + + def testGraphModeCaptureVariable(self): + with context.graph_mode(), self.test_session() as sess: + + class HasAVar(object): + + def __init__(self): + self.v = resource_variable_ops.ResourceVariable(1.0) + + def call(self): + return self.v * 2 + + o = HasAVar() + variables.global_variables_initializer().run() + call = function.defun(o.call) + op = call() + self.assertAllEqual(sess.run(op), 2.0) + + def testGraphModeManyFunctions(self): + with context.graph_mode(), self.test_session(): + + @function.defun + def f(x): + return x * x + + @function.defun + def g(x): + return f(x) + 1 + + self.assertAllEqual(g(constant_op.constant(2.0)).eval(), 5.0) + + def testDict(self): + + @function.defun + def f(x): + return {'name': x + 1} + + self.assertAllEqual(f(constant_op.constant(1.0))['name'], 2.0) def testTensorConversionWithDefun(self): @@ -73,7 +113,7 @@ class FunctionTest(test.TestCase): def f(x): return math_ops.add(x, constant_op.constant(3)) - self.assertAllEqual(5, f(constant_op.constant(2)).numpy()) + self.assertAllEqual(5, f(constant_op.constant(2))) def testTensorConversionCall(self): @@ -85,7 +125,7 @@ class FunctionTest(test.TestCase): def g(x): return f(f(x)) - self.assertAllEqual(8, g(constant_op.constant(2)).numpy()) + self.assertAllEqual(8, g(constant_op.constant(2))) def testDefunCallBackprop(self): @@ -97,7 +137,17 @@ class FunctionTest(test.TestCase): def g(x): return backprop.gradients_function(f, [0])(x)[0] - self.assertAllEqual(2, g(constant_op.constant(2)).numpy()) + self.assertAllEqual(2, g(constant_op.constant(2))) + + def testGraphModeEagerGradError(self): + with context.graph_mode(): + def f(): + x = variable_scope.get_variable( + 'v', initializer=constant_op.constant(1.0)) + return x * constant_op.constant(2.0) + with self.assertRaisesRegexp(ValueError, + 'No trainable variables were accessed'): + backprop.implicit_val_and_grad(f)() def testDefunCallBackpropUsingSameObjectForMultipleArguments(self): @@ -127,7 +177,7 @@ class FunctionTest(test.TestCase): g(constant_op.constant(1.0)) def testGradientTensorConversionWithDefun(self): - three = resource_variable_ops.ResourceVariable(3.0) + three = resource_variable_ops.ResourceVariable(3.0, name='v') @function.defun def f(x): @@ -138,7 +188,7 @@ class FunctionTest(test.TestCase): return f(x) g = backprop.implicit_grad(g)(constant_op.constant(1.0))[0][0] - self.assertEqual(g.numpy(), 1.0) + self.assertAllEqual(g, 1.0) def testGradient(self): matmul = function.defun(math_ops.matmul) @@ -148,7 +198,7 @@ class FunctionTest(test.TestCase): t = constant_op.constant([[1.0, 2.0], [3.0, 4.0]]) grad_t, = backprop.gradients_function(sq, [0])(t) - self.assertAllEqual(grad_t.numpy(), [[6, 6], [14, 14]]) + self.assertAllEqual(grad_t, [[6, 6], [14, 14]]) def testGradientInFunction(self): @@ -156,16 +206,16 @@ class FunctionTest(test.TestCase): def f(x): return backprop.gradients_function(lambda y: y * y, [0])(x)[0] - self.assertEqual(f(constant_op.constant(1.0)).numpy(), 2.0) + self.assertAllEqual(f(constant_op.constant(1.0)), 2.0) def testFunctionOnDevice(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') - x = constant_op.constant([1.]).as_gpu_tensor() + x = constant_op.constant([1.]).gpu() f = function.defun(math_ops.add) - y = f(x, x).as_cpu_tensor() - self.assertAllEqual(y.numpy(), [2.]) + y = f(x, x).cpu() + self.assertAllEqual(y, [2.]) def testFunctionHandlesInputsOnDifferentDevices(self): if not context.context().num_gpus(): @@ -173,10 +223,10 @@ class FunctionTest(test.TestCase): # The Reshape op requires the shape tensor to be placed in host memory. reshape = function.defun(array_ops.reshape) - value = constant_op.constant([1., 2.]).as_gpu_tensor() + value = constant_op.constant([1., 2.]).gpu() shape = constant_op.constant([2, 1]) - reshaped = reshape(value, shape).as_cpu_tensor() - self.assertAllEqual(reshaped.numpy(), [[1], [2]]) + reshaped = reshape(value, shape).cpu() + self.assertAllEqual(reshaped, [[1], [2]]) def testFunctionHandlesInputsPlacedOnTheWrongDeviceGracefully(self): if not context.context().num_gpus(): @@ -184,8 +234,8 @@ class FunctionTest(test.TestCase): # The Reshape op requires the shape tensor to be placed in host memory. reshape = function.defun(array_ops.reshape) - value = constant_op.constant([1., 2.]).as_gpu_tensor() - shape = constant_op.constant([2, 1]).as_gpu_tensor() + value = constant_op.constant([1., 2.]).gpu() + shape = constant_op.constant([2, 1]).gpu() with self.assertRaises(errors.InvalidArgumentError): reshape(value, shape) @@ -199,7 +249,7 @@ class FunctionTest(test.TestCase): return my_function(x)[0] g = backprop.gradients_function(wrapper, [0])(constant_op.constant(0.0)) - self.assertAllEqual(g[0].numpy(), 1.) + self.assertAllEqual(g[0], 1.) def testNoneOutput(self): @@ -220,7 +270,7 @@ class FunctionTest(test.TestCase): def add_one(x): return add(x, 1) - self.assertAllEqual(3, add_one(constant_op.constant(2)).numpy()) + self.assertAllEqual(3, add_one(constant_op.constant(2))) def testSequenceInputs(self): clip_by_global_norm = function.defun(clip_ops.clip_by_global_norm) @@ -247,13 +297,13 @@ class FunctionTest(test.TestCase): constant_op.constant(5) ]) self.assertEqual(len(ret), 2) - self.assertEqual(ret[0][0].numpy(), 2) - self.assertEqual(ret[0][1][0][0].numpy(), 8) - self.assertEqual(ret[0][1][0][1].numpy(), 4) + self.assertAllEqual(ret[0][0], 2) + self.assertAllEqual(ret[0][1][0][0], 8) + self.assertAllEqual(ret[0][1][0][1], 4) self.assertTrue(isinstance(ret[0][1][0], tuple)) - self.assertEqual(ret[0][1][1].numpy(), 6) - self.assertEqual(ret[0][2].numpy(), 10) - self.assertEqual(ret[1].numpy(), 15) + self.assertAllEqual(ret[0][1][1], 6) + self.assertAllEqual(ret[0][2], 10) + self.assertAllEqual(ret[1], 15) if __name__ == '__main__': diff --git a/tensorflow/python/eager/graph_callable.py b/tensorflow/python/eager/graph_callable.py index 3aba164630d7968b37c09f9bf69518b615f84f70..a7f1061d18bf905caf97decc5375c3996215ec5b 100644 --- a/tensorflow/python/eager/graph_callable.py +++ b/tensorflow/python/eager/graph_callable.py @@ -45,28 +45,6 @@ def _default_initializer(name, shape, dtype): return initializer[0] -class _VariableFromResource(resource_variable_ops.ResourceVariable): - """Variable object from a preexisting resource. - - Required because the ResourceVariable constructor creates the resource handle, - and here we want to use a preexisting one. - """ - - def __init__(self, resource, dtype, name, shape): - self._handle = resource - self._graph_shape = tensor_shape.as_shape(shape) - self._handle_device = resource.device - self._handle_name = name - self._cached_value = None - self._initializer_op = None - self._caching_device = None - self._dtype = dtype - self._constraint = None - self._in_graph_mode = context.in_graph_mode() - if self._in_graph_mode: - self._graph_element = self.read_value() - - class _CapturedVariable(object): """Variable captured by graph_callable. @@ -137,17 +115,11 @@ class _VariableCapturingScope(object): trainable=True, collections=None, caching_device=None, # pylint: disable=redefined-outer-name partitioner=None, validate_shape=True, use_resource=None): - del getter, regularizer, partitioner, validate_shape, use_resource - del collections, initializer, trainable, reuse, caching_device + del getter, regularizer, partitioner, validate_shape, use_resource, dtype + del collections, initializer, trainable, reuse, caching_device, shape, assert name in self.variables v = self.variables[name] - v.placeholder = array_ops.placeholder(dtype=dtypes.resource, shape=shape) - # TODO(apassos) remove the need for this by correctly dealing with shape - # inference. - v.placeholder._handle_data = v.variable.handle._handle_data # pylint: disable=protected-access - return _VariableFromResource( - v.placeholder, dtype=dtypes.as_dtype(dtype), name=name, - shape=v.shape) + return v.variable scope = variable_scope.get_variable_scope() with variable_scope.variable_scope(scope, custom_getter=_custom_getter): @@ -181,14 +153,12 @@ class _VariableCapturingScope(object): v = _CapturedVariable(name, initializer, shape, dtype, trainable) self.variables[name] = v - graph_mode_resource = resource_variable_ops.var_handle_op( - shared_name=name, shape=shape, dtype=dtype) + graph_mode_resource = v.variable.handle if initializer is None: initializer = _default_initializer(name, shape, dtype) - resource_variable_ops.assign_variable_op( - graph_mode_resource, initializer(shape, dtype)) - return _VariableFromResource( - graph_mode_resource, dtype, name, shape=v.shape) + resource_variable_ops.shape_safe_assign_variable_handle( + graph_mode_resource, v.variable.shape, initializer(shape, dtype)) + return v.variable scope = variable_scope.get_variable_scope() with variable_scope.variable_scope(scope, custom_getter=_custom_getter): @@ -220,13 +190,6 @@ class _FunctionObject(function._GraphModeFunction): # pylint: disable=protected def variables(self): return [x.variable for x in self._variables] - def __call__(self, *args, **kwds): - kwds.pop("want_gradients", False) - if kwds: - raise ValueError("graph_callable functions do not take keyword args") - values = [x.variable.handle for x in self._variables] - return super(_FunctionObject, self).__call__(*(values + list(args))) - class _InitializingFunctionObject(object): """Responsible for deciding which version of func-to-object to call. @@ -312,11 +275,22 @@ def _graph_callable_internal(func, shape_and_dtypes): Returns: Callable graph object. """ + container = tf_ops.get_default_graph()._container # pylint: disable=protected-access + container_prefix = tf_ops.get_default_graph()._container_prefix # pylint: disable=protected-access with context.graph_mode(): # This graph will store both the initialization and the call version of the # wrapped function. It will later be used by the backprop code to build the # backprop graph, if necessary. - tmp_graph = tf_ops.Graph() + captures = {} + tmp_graph = function.CapturingGraph(captures) + # Inherit the container from the original graph to create resources at user + # expected containers. Also inherits the container prefix, since this is + # used for error checking when isolating Eager execution (the container + # prefix at creation must match the container prefix when used, and + # variables returned from the graph callable will be used in the outside + # context). + tmp_graph._container = container # pylint: disable=protected-access + tmp_graph._container_prefix = container_prefix # pylint: disable=protected-access with tmp_graph.as_default(): # Placeholders for the non-variable inputs. func_inputs = _get_graph_callable_inputs(shape_and_dtypes) @@ -332,7 +306,6 @@ def _graph_callable_internal(func, shape_and_dtypes): # variables. As a side-effect this will populate the variable capturing # scope's view of which variables exist. variable_captures = _VariableCapturingScope() - captures = {} with variable_captures.initializing_scope(), function.capture_tensors( captures): func_outputs = func(*func_inputs) @@ -356,7 +329,6 @@ def _graph_callable_internal(func, shape_and_dtypes): sorted_variables = sorted(variable_captures.variables.values(), key=lambda x: x.name) - variable_placeholders = [x.placeholder for x in sorted_variables] ids = list(sorted(captures.keys())) if ids: extra_inputs, extra_placeholders = zip(*[captures[x] for x in ids]) @@ -367,7 +339,6 @@ def _graph_callable_internal(func, shape_and_dtypes): flat_inputs = [x for x in nest.flatten(func_inputs) if isinstance(x, tf_ops.Tensor)] placeholder_inputs = flat_inputs+ list(extra_placeholders) - all_inputs = variable_placeholders + placeholder_inputs func_def_outputs = [x for x in outputs_list if isinstance(x, tf_ops.Tensor)] initializer_function_def = function.make_function_def( @@ -397,13 +368,13 @@ def _graph_callable_internal(func, shape_and_dtypes): captured_function_def = function.make_function_def( tmp_graph, capturing_operations, - all_inputs, + placeholder_inputs, capture_func_def_outputs) function._register_with_name(function._inference_name(func.__name__), # pylint: disable=protected-access captured_function_def) captured_function = _FunctionObject( sorted_variables, - all_inputs, + placeholder_inputs, extra_inputs, captured_function_def, tmp_graph, diff --git a/tensorflow/python/eager/graph_callable_test.py b/tensorflow/python/eager/graph_callable_test.py index e77a33981d1155f8b04a9ae3ae956846dbe212dd..548e16a909f8fe846ea6d5a7a33c4247c5d90054 100644 --- a/tensorflow/python/eager/graph_callable_test.py +++ b/tensorflow/python/eager/graph_callable_test.py @@ -57,7 +57,7 @@ class GraphCallableTest(test.TestCase): v.assign(x) my_function(constant_op.constant(4, dtype=dtypes.float32)) - self.assertEqual(4, my_function.variables[0].read_value().numpy()) + self.assertAllEqual(4, my_function.variables[0].read_value()) def testFunctionWithoutReturnValueAndArgs(self): @@ -68,7 +68,7 @@ class GraphCallableTest(test.TestCase): v.assign(4) my_function() - self.assertEqual(4, my_function.variables[0].read_value().numpy()) + self.assertAllEqual(4, my_function.variables[0].read_value()) def testVariableAPI(self): @@ -113,7 +113,7 @@ class GraphCallableTest(test.TestCase): v.assign(v * x) return v.read_value() - self.assertEqual(my_function(constant_op.constant(2.0)).numpy(), 6.0) + self.assertAllEqual(my_function(constant_op.constant(2.0)), 6.0) def testEmptyInitializer(self): @@ -149,7 +149,7 @@ class GraphCallableTest(test.TestCase): def f(x): return math_ops.add(x, constant_op.constant(3)) - self.assertAllEqual(5, f(constant_op.constant(2)).numpy()) + self.assertAllEqual(5, f(constant_op.constant(2))) def testNestedFunction(self): @@ -165,7 +165,7 @@ class GraphCallableTest(test.TestCase): def add_one(x): return add(x, 1) - self.assertAllEqual(3, add_one(constant_op.constant(2)).numpy()) + self.assertAllEqual(3, add_one(constant_op.constant(2))) # TODO(ashankar): Make this work. # The problem is that the two graph_callables (for add_one and add_two) @@ -187,8 +187,8 @@ class GraphCallableTest(test.TestCase): return add(x, 2) two = constant_op.constant(2) - self.assertAllEqual(3, add_one(two).numpy()) - self.assertAllEqual(4, add_two(two).numpy()) + self.assertAllEqual(3, add_one(two)) + self.assertAllEqual(4, add_two(two)) def testNestedSequenceInputs(self): sd = graph_callable.ShapeAndDtype(shape=(), dtype=dtypes.float32) @@ -205,11 +205,11 @@ class GraphCallableTest(test.TestCase): constant_op.constant(4.)] ret = my_op(inputs) self.assertEqual(len(ret), 2.) - self.assertEqual(ret[1].numpy(), 10.) + self.assertAllEqual(ret[1], 10.) my_op.variables[0].assign(1.) ret = my_op(inputs) - self.assertEqual(ret[1].numpy(), 11.) + self.assertAllEqual(ret[1], 11.) def testVariableShapeIsTensorShape(self): @graph_callable.graph_callable([]) @@ -243,7 +243,7 @@ class GraphCallableTest(test.TestCase): grad_fn = backprop.implicit_grad(my_function) grads_and_vars = list(zip(*grad_fn())) - self.assertEqual(6., grads_and_vars[0][0].numpy()) + self.assertAllEqual(6., grads_and_vars[0][0]) if __name__ == "__main__": diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py index d30d124040d0bc45821697ed065b90ddd15b4fdf..c87719f84abf22f4dee775ab61309d1b18129e07 100644 --- a/tensorflow/python/eager/imperative_grad.py +++ b/tensorflow/python/eager/imperative_grad.py @@ -120,7 +120,14 @@ def _initial_gradients(vspace, target, output_gradients, tensor_usage_counts): VSpace = collections.namedtuple( "VSpace", - ["add_new_grads_fn", "aggregate_fn", "tensor_id", "zeros", "ones_like"]) + ["aggregate_fn", "num_elements_fn", "tensor_id", "zeros", "ones_like"]) + + +# If over MIN_AGGREGATE_COUNT gradients are accumulated and the total +# memory consumption is over MIN_AGGREGATE_BYTES, do an early aggregation +# so as to release the gradient tensor to save memory. +_MIN_AGGREGATE_COUNT = 4 +_MIN_AGGREGATE_BYTES = 128 * 1024 * 1024 def imperative_grad( @@ -193,14 +200,22 @@ def imperative_grad( in_gradients = op_trace.backward_function(*(out_gradients)) for i, t in enumerate(op_trace.input_ids): if in_gradients[i] is not None: - vspace.add_new_grads_fn(gradients, gradients_size, t, in_gradients[i]) + t_grads = gradients.setdefault(t, []) + t_grads.append(in_gradients[i]) + if len(t_grads) >= _MIN_AGGREGATE_COUNT: + if t not in gradients_size: + gradients_size[t] = vspace.num_elements_fn(t_grads[-1]) + size = gradients_size[t] + + if len(t_grads) * size * 4 > _MIN_AGGREGATE_BYTES: + t_grads[:] = [vspace.aggregate_fn(t_grads)] if tensor_usage_counts.get(t, 0) > 0: tensor_usage_counts[t] -= 1 if (t in tensor_to_op and tensor_usage_counts[t] == 0 and t not in id_sources): in_op = tensor_to_op[t] - if in_op is None: + if in_op is None or in_op == -1: continue if op_missing_tensor.get(in_op, 0) > 0: op_missing_tensor[in_op] -= 1 diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py index 7d54b8d2d869077565cf075ade2e5e81484eb8cd..e86073d6b21e031ea4974f514e1401fd0211c962 100644 --- a/tensorflow/python/eager/ops_test.py +++ b/tensorflow/python/eager/ops_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import test_util from tensorflow.python.layers import core from tensorflow.python.ops import array_ops @@ -40,13 +41,13 @@ class OpsTest(test_util.TensorFlowTestCase): three = constant_op.constant(3) five = constant_op.constant(5) product = three * five - self.assertEqual(15, product.numpy()) + self.assertAllEqual(15, product) def testMatMulGPU(self): if not context.context().num_gpus(): self.skipTest('No GPUs found') - three = constant_op.constant([[3.]]).as_gpu_tensor() - five = constant_op.constant([[5.]]).as_gpu_tensor() + three = constant_op.constant([[3.]]).gpu() + five = constant_op.constant([[5.]]).gpu() product = math_ops.matmul(three, five) self.assertEqual([[15.0]], product.numpy()) @@ -61,27 +62,27 @@ class OpsTest(test_util.TensorFlowTestCase): almost_three = constant_op.constant(2.8) almost_equal = math_ops.approximate_equal( three, almost_three, tolerance=0.3) - self.assertTrue(almost_equal.numpy()) + self.assertTrue(almost_equal) def testExecuteIntAttr(self): three = constant_op.constant(3) four = constant_op.constant(4) total = math_ops.add_n([three, four]) - self.assertEqual(7, total.numpy()) + self.assertAllEqual(7, total) def testExecuteBoolAttr(self): three = constant_op.constant([[3]]) five = constant_op.constant([[5]]) product = math_ops.matmul(three, five, transpose_a=True) - self.assertEqual([[15]], product.numpy()) + self.assertAllEqual([[15]], product) def testExecuteOneListOutput(self): split_dim = constant_op.constant(1) value = constant_op.constant([[0, 1, 2], [3, 4, 5]]) x1, x2, x3 = array_ops.split(value, 3, axis=split_dim) - self.assertAllEqual([[0], [3]], x1.numpy()) - self.assertAllEqual([[1], [4]], x2.numpy()) - self.assertAllEqual([[2], [5]], x3.numpy()) + self.assertAllEqual([[0], [3]], x1) + self.assertAllEqual([[1], [4]], x2) + self.assertAllEqual([[2], [5]], x3) def testGraphMode(self): graph = ops.Graph() @@ -96,7 +97,7 @@ class OpsTest(test_util.TensorFlowTestCase): self.skipTest('No GPUs found') with context.device('/gpu:0'): r = constant_op.constant(1) + constant_op.constant(2) - self.assertEqual(r.numpy(), 3) + self.assertAllEqual(r, 3) def testExecuteListOutputLen1(self): split_dim = constant_op.constant(1) @@ -104,7 +105,7 @@ class OpsTest(test_util.TensorFlowTestCase): result = array_ops.split(value, 1, axis=split_dim) self.assertTrue(isinstance(result, list)) self.assertEqual(1, len(result)) - self.assertAllEqual([[0, 1, 2], [3, 4, 5]], result[0].numpy()) + self.assertAllEqual([[0, 1, 2], [3, 4, 5]], result[0]) def testExecuteListOutputLen0(self): empty = constant_op.constant([], dtype=dtypes.int32) @@ -119,8 +120,8 @@ class OpsTest(test_util.TensorFlowTestCase): out, idx = result self.assertTrue(out is result.out) self.assertTrue(idx is result.idx) - self.assertAllEqual([2, 4, 6], out.numpy()) - self.assertAllEqual([1, 3, 5], idx.numpy()) + self.assertAllEqual([2, 4, 6], out) + self.assertAllEqual([1, 3, 5], idx) def testExecuteMultipleListOutput(self): split_dim = constant_op.constant(1, dtype=dtypes.int64) @@ -137,12 +138,12 @@ class OpsTest(test_util.TensorFlowTestCase): self.assertEqual(output_indices, result.output_indices) self.assertEqual(output_values, result.output_values) self.assertEqual(output_shape, result.output_shape) - self.assertAllEqual([[0, 2], [1, 0], [1, 1]], output_indices[0].numpy()) - self.assertAllEqual([[0, 0], [0, 1]], output_indices[1].numpy()) - self.assertAllEqual([2, 7, 11], output_values[0].numpy()) - self.assertAllEqual([3, 5], output_values[1].numpy()) - self.assertAllEqual([2, 4], output_shape[0].numpy()) - self.assertAllEqual([2, 3], output_shape[1].numpy()) + self.assertAllEqual([[0, 2], [1, 0], [1, 1]], output_indices[0]) + self.assertAllEqual([[0, 0], [0, 1]], output_indices[1]) + self.assertAllEqual([2, 7, 11], output_values[0]) + self.assertAllEqual([3, 5], output_values[1]) + self.assertAllEqual([2, 4], output_shape[0]) + self.assertAllEqual([2, 3], output_shape[1]) # TODO(josh11b): Test an op that has multiple outputs, some but not # all of which are lists. Examples: barrier_take_many (currently @@ -153,84 +154,84 @@ class OpsTest(test_util.TensorFlowTestCase): x = constant_op.constant(1, dtype=dtypes.int32) three_x = x + x + x self.assertEquals(dtypes.int32, three_x.dtype) - self.assertEquals(3, three_x.numpy()) + self.assertAllEqual(3, three_x) def testOperatorOverrides(self): # TODO(henrytan): test with negative number. a = constant_op.constant([1]) b = constant_op.constant([2]) - self.assertAllEqual((-a).numpy(), [-1]) - self.assertAllEqual(abs(b).numpy(), [2]) + self.assertAllEqual((-a), [-1]) + self.assertAllEqual(abs(b), [2]) - self.assertAllEqual((a + b).numpy(), [3]) - self.assertAllEqual((a - b).numpy(), [-1]) - self.assertAllEqual((a * b).numpy(), [2]) - self.assertAllEqual((a * a).numpy(), [1]) + self.assertAllEqual((a + b), [3]) + self.assertAllEqual((a - b), [-1]) + self.assertAllEqual((a * b), [2]) + self.assertAllEqual((a * a), [1]) - self.assertAllEqual((a**b).numpy(), [1]) - self.assertAllEqual((a / b).numpy(), [1 / 2]) - self.assertAllEqual((a / a).numpy(), [1]) - self.assertAllEqual((a % b).numpy(), [1]) + self.assertAllEqual((a**b), [1]) + self.assertAllEqual((a / b), [1 / 2]) + self.assertAllEqual((a / a), [1]) + self.assertAllEqual((a % b), [1]) - self.assertAllEqual((a < b).numpy(), [True]) - self.assertAllEqual((a <= b).numpy(), [True]) - self.assertAllEqual((a > b).numpy(), [False]) - self.assertAllEqual((a >= b).numpy(), [False]) + self.assertAllEqual((a < b), [True]) + self.assertAllEqual((a <= b), [True]) + self.assertAllEqual((a > b), [False]) + self.assertAllEqual((a >= b), [False]) self.assertAllEqual((a == b), False) self.assertAllEqual((a != b), True) - self.assertEqual(1, a[constant_op.constant(0)].numpy()) + self.assertAllEqual(1, a[constant_op.constant(0)]) def test_basic_slice(self): npt = np.arange(1, 19, dtype=np.float32).reshape(3, 2, 3) t = constant_op.constant(npt) - self.assertAllEqual(npt[:, :, :], t[:, :, :].numpy()) - self.assertAllEqual(npt[::, ::, ::], t[::, ::, ::].numpy()) - self.assertAllEqual(npt[::1, ::1, ::1], t[::1, ::1, ::1].numpy()) - self.assertAllEqual(npt[::1, ::5, ::2], t[::1, ::5, ::2].numpy()) - self.assertAllEqual(npt[::-1, :, :], t[::-1, :, :].numpy()) - self.assertAllEqual(npt[:, ::-1, :], t[:, ::-1, :].numpy()) - self.assertAllEqual(npt[:, :, ::-1], t[:, :, ::-1].numpy()) - self.assertAllEqual(npt[-2::-1, :, ::1], t[-2::-1, :, ::1].numpy()) - self.assertAllEqual(npt[-2::-1, :, ::2], t[-2::-1, :, ::2].numpy()) + self.assertAllEqual(npt[:, :, :], t[:, :, :]) + self.assertAllEqual(npt[::, ::, ::], t[::, ::, ::]) + self.assertAllEqual(npt[::1, ::1, ::1], t[::1, ::1, ::1]) + self.assertAllEqual(npt[::1, ::5, ::2], t[::1, ::5, ::2]) + self.assertAllEqual(npt[::-1, :, :], t[::-1, :, :]) + self.assertAllEqual(npt[:, ::-1, :], t[:, ::-1, :]) + self.assertAllEqual(npt[:, :, ::-1], t[:, :, ::-1]) + self.assertAllEqual(npt[-2::-1, :, ::1], t[-2::-1, :, ::1]) + self.assertAllEqual(npt[-2::-1, :, ::2], t[-2::-1, :, ::2]) def testDegenerateSlices(self): npt = np.arange(1, 19, dtype=np.float32).reshape(3, 2, 3) t = constant_op.constant(npt) # degenerate by offering a forward interval with a negative stride - self.assertAllEqual(npt[0:-1:-1, :, :], t[0:-1:-1, :, :].numpy()) + self.assertAllEqual(npt[0:-1:-1, :, :], t[0:-1:-1, :, :]) # degenerate with a reverse interval with a positive stride - self.assertAllEqual(npt[-1:0, :, :], t[-1:0, :, :].numpy()) + self.assertAllEqual(npt[-1:0, :, :], t[-1:0, :, :]) # empty interval in every dimension - self.assertAllEqual(npt[-1:0, 2:2, 2:3:-1], t[-1:0, 2:2, 2:3:-1].numpy()) + self.assertAllEqual(npt[-1:0, 2:2, 2:3:-1], t[-1:0, 2:2, 2:3:-1]) def testEllipsis(self): npt = np.array( [[[[[1, 2], [3, 4], [5, 6]]], [[[7, 8], [9, 10], [11, 12]]]]]) t = constant_op.constant(npt) - self.assertAllEqual(npt[0:], t[0:].numpy()) + self.assertAllEqual(npt[0:], t[0:]) # implicit ellipsis - self.assertAllEqual(npt[0:, ...], t[0:, ...].numpy()) + self.assertAllEqual(npt[0:, ...], t[0:, ...]) # ellipsis alone - self.assertAllEqual(npt[...], t[...].numpy()) + self.assertAllEqual(npt[...], t[...]) # ellipsis at end - self.assertAllEqual(npt[0:1, ...], t[0:1, ...].numpy()) + self.assertAllEqual(npt[0:1, ...], t[0:1, ...]) # ellipsis at begin - self.assertAllEqual(npt[..., 0:1], t[..., 0:1].numpy()) + self.assertAllEqual(npt[..., 0:1], t[..., 0:1]) # ellipsis at middle - self.assertAllEqual(npt[0:1, ..., 0:1], t[0:1, ..., 0:1].numpy()) + self.assertAllEqual(npt[0:1, ..., 0:1], t[0:1, ..., 0:1]) def testShrink(self): npt = np.array([[[[[1, 2, 4, 5], [5, 6, 7, 8], [9, 10, 11, 12]]], [[[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]]]]]) t = constant_op.constant(npt) - self.assertAllEqual(npt[:, :, :, :, 3], t[:, :, :, :, 3].numpy()) - self.assertAllEqual(npt[..., 3], t[..., 3].numpy()) - self.assertAllEqual(npt[:, 0], t[:, 0].numpy()) - self.assertAllEqual(npt[:, :, 0], t[:, :, 0].numpy()) + self.assertAllEqual(npt[:, :, :, :, 3], t[:, :, :, :, 3]) + self.assertAllEqual(npt[..., 3], t[..., 3]) + self.assertAllEqual(npt[:, 0], t[:, 0]) + self.assertAllEqual(npt[:, :, 0], t[:, :, 0]) def testOpWithInputsOnDifferentDevices(self): if not context.context().num_gpus(): @@ -238,18 +239,19 @@ class OpsTest(test_util.TensorFlowTestCase): # The GPU kernel for the Reshape op requires that the # shape input be on CPU. - value = constant_op.constant([1., 2.]).as_gpu_tensor() + value = constant_op.constant([1., 2.]).gpu() shape = constant_op.constant([2, 1]) reshaped = array_ops.reshape(value, shape) - self.assertAllEqual([[1], [2]], reshaped.as_cpu_tensor().numpy()) + self.assertAllEqual([[1], [2]], reshaped.cpu()) # And if the shape is in device memory, it should complain # TODO(ashankar): Revisit this - perhaps instead of complaining, # it should implicitly copy the tensor to host memory? with self.assertRaisesRegexp( errors.InvalidArgumentError, - 'cannot compute Reshape as input #1 was expected to be on'): - reshaped = array_ops.reshape(value, shape.as_gpu_tensor()) + 'cannot compute Reshape as input #1 was expected to be on.*' + 'using.*DEVICE_PLACEMENT_SILENT'): + reshaped = array_ops.reshape(value, shape.gpu()) def testInvalidInputDataType(self): # Fill requires the first input to be an int32 tensor. @@ -261,9 +263,26 @@ class OpsTest(test_util.TensorFlowTestCase): if not context.context().num_gpus(): self.skipTest('No GPUs found') # The Shape op kernel on GPU places the output in host memory. - value = constant_op.constant([1.]).as_gpu_tensor() + value = constant_op.constant([1.]).gpu() shape = array_ops.shape(value) - self.assertEquals([1], shape.numpy()) + self.assertEqual([1], shape.numpy()) + + def testSilentCopy(self): + if not context.context().num_gpus(): + self.skipTest('No GPUs found') + # Temporarily replace the context + # pylint: disable=protected-access + del context._context + try: + context._context = context.Context( + device_policy=context.DEVICE_PLACEMENT_SILENT) + cpu_tensor = constant_op.constant(1.0) + gpu_tensor = cpu_tensor.gpu() + self.assertAllEqual(cpu_tensor + gpu_tensor, 2.0) + finally: + del context._context + context._context = context.Context() + # pylint: enable=protected-access def testRandomUniform(self): scalar_shape = constant_op.constant([], dtype=dtypes.int32) @@ -275,8 +294,8 @@ class OpsTest(test_util.TensorFlowTestCase): x = random_ops.random_uniform( scalar_shape, minval=constant_op.constant(5.), maxval=constant_op.constant(6.)) - self.assertLess(x.numpy(), 6) - self.assertGreaterEqual(x.numpy(), 5) + self.assertLess(x, 6) + self.assertGreaterEqual(x, 5) def testArgsToMatchingEagerDefault(self): # Uses default @@ -297,10 +316,34 @@ class OpsTest(test_util.TensorFlowTestCase): flatten_layer = core.Flatten() x = constant_op.constant([[[-10, -20], [-30, -40]], [[10, 20], [30, 40]]]) y = flatten_layer(x) - self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y.numpy()) + self.assertAllEqual([[-10, -20, -30, -40], [10, 20, 30, 40]], y) def testIdentity(self): - self.assertEqual(2, array_ops.identity(2).numpy()) + self.assertAllEqual(2, array_ops.identity(2)) + + def testIncompatibleSetShape(self): + x = constant_op.constant(1) + with self.assertRaises(ValueError): + x.set_shape((1, 2)) + + def testCompatibleSetShape(self): + x = constant_op.constant([[1, 2]]) + x.set_shape(tensor_shape.TensorShape([None, 2])) + self.assertEqual(x.get_shape(), (1, 2)) + + def testCastScalarToPrimitiveTypes(self): + x = constant_op.constant(1.3) + self.assertIsInstance(int(x), int) + self.assertEqual(int(x), 1) + self.assertIsInstance(float(x), float) + self.assertAllClose(float(x), 1.3) + + def testCastNonScalarToPrimitiveTypesFails(self): + x = constant_op.constant([1.3, 2]) + with self.assertRaises(TypeError): + int(x) + with self.assertRaises(TypeError): + float(x) if __name__ == '__main__': diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc index 157e87d3876ba4e9702dbe468dab3f60ea716d1f..3adaea2c7913be134b0573780ddb881c219604e0 100644 --- a/tensorflow/python/eager/pywrap_tensor.cc +++ b/tensorflow/python/eager/pywrap_tensor.cc @@ -377,6 +377,15 @@ static PyObject* EagerTensor_shape_tuple(EagerTensor* self) { return shape; } +// Getter for `_rank`. +static PyObject* EagerTensor_rank(EagerTensor* self) { +#if PY_MAJOR_VERSION < 3 + return PyInt_FromLong(TFE_TensorHandleNumDims(self->handle)); +#else + return PyLong_FromLong(TFE_TensorHandleNumDims(self->handle)); +#endif +} + static PyObject* EagerTensor_tensor_handle(EagerTensor* self, void* unused) { Py_INCREF(self->handle_data); return self->handle_data; @@ -470,6 +479,7 @@ static PyMethodDef EagerTensor_methods[] = { PyDoc_STR("_datatype_enum")}, {"_shape_tuple", (PyCFunction)EagerTensor_shape_tuple, METH_NOARGS, PyDoc_STR("_shape_tuple")}, + {"_rank", (PyCFunction)EagerTensor_rank, METH_NOARGS, PyDoc_STR("_rank")}, {"_copy_to_device", (PyCFunction)EagerTensor_copy_to_device, METH_VARARGS | METH_KEYWORDS, PyDoc_STR("_copy_to_device")}, {nullptr, nullptr}, diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc index 402b84d7c6e2a92b8a8ea3292f0130ab3e6c2973..7456eb10f867e797e32e314159b70b3e06b3d01d 100644 --- a/tensorflow/python/eager/pywrap_tfe_src.cc +++ b/tensorflow/python/eager/pywrap_tfe_src.cc @@ -452,26 +452,26 @@ static void TFE_Py_Tape_Delete(PyObject* tape) { } static PyTypeObject TFE_Py_Tape_Type = { - PyVarObject_HEAD_INIT(NULL, 0) "tfe.Tape", /* tp_name */ - sizeof(TFE_Py_Tape), /* tp_basicsize */ - 0, /* tp_itemsize */ - &TFE_Py_Tape_Delete, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "TFE_Py_Tape objects", /* tp_doc */ + PyVarObject_HEAD_INIT(nullptr, 0) "tfe.Tape", /* tp_name */ + sizeof(TFE_Py_Tape), /* tp_basicsize */ + 0, /* tp_itemsize */ + &TFE_Py_Tape_Delete, /* tp_dealloc */ + nullptr, /* tp_print */ + nullptr, /* tp_getattr */ + nullptr, /* tp_setattr */ + nullptr, /* tp_reserved */ + nullptr, /* tp_repr */ + nullptr, /* tp_as_number */ + nullptr, /* tp_as_sequence */ + nullptr, /* tp_as_mapping */ + nullptr, /* tp_hash */ + nullptr, /* tp_call */ + nullptr, /* tp_str */ + nullptr, /* tp_getattro */ + nullptr, /* tp_setattro */ + nullptr, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "TFE_Py_Tape objects", /* tp_doc */ }; PyObject* TFE_Py_NewTape() { diff --git a/tensorflow/python/eager/tape_test.py b/tensorflow/python/eager/tape_test.py index c34f5cffe3671af222d6c40c423aae132b2a4848..c97cb62125741ccdec495d925651a3559bd5fb9c 100644 --- a/tensorflow/python/eager/tape_test.py +++ b/tensorflow/python/eager/tape_test.py @@ -81,8 +81,8 @@ class TapeTest(test.TestCase): tf_e = tf_d + tf_f tf_da, tf_db = gradients_impl.gradients(tf_e, [tf_a, tf_b]) - self.assertAllEqual(da.numpy(), tf_da.eval()) - self.assertAllEqual(db.numpy(), tf_db.eval()) + self.assertAllEqual(da, tf_da.eval()) + self.assertAllEqual(db, tf_db.eval()) def testBasicFunctional(self): @@ -93,7 +93,7 @@ class TapeTest(test.TestCase): aa = constant_op.constant([[1., 0.], [0., 1.]]) bb = constant_op.constant([[1., 2.], [3., 4.]]) da, = backprop.gradients_function(forward, ['a'])(aa, bb) - self.assertAllEqual(da.numpy(), + self.assertAllEqual(da, math_ops.matmul( array_ops.ones_like(aa), array_ops.transpose(bb)).numpy()) @@ -107,7 +107,7 @@ class TapeTest(test.TestCase): aa = constant_op.constant([[1., 0.], [0., 1.]]) bb = constant_op.constant([[1., 2.], [3., 4.]]) da, = backprop.gradients_function(forward, [0])(aa, bb) - self.assertAllEqual(da.numpy(), + self.assertAllEqual(da, math_ops.matmul( array_ops.ones_like(aa), array_ops.transpose(bb)).numpy()) @@ -121,11 +121,11 @@ class TapeTest(test.TestCase): aa = constant_op.constant([[1., 0.], [0., 1.]]) bb = constant_op.constant([[1., 2.], [3., 4.]]) val, (da,) = backprop.val_and_grad_function(forward, ['a'])(aa, bb) - self.assertAllEqual(da.numpy(), + self.assertAllEqual(da, math_ops.matmul( array_ops.ones_like(aa), - array_ops.transpose(bb)).numpy()) - self.assertAllEqual(val.numpy(), forward(aa, bb).numpy()) + array_ops.transpose(bb))) + self.assertAllEqual(val, forward(aa, bb)) def testTwoOutputs(self): @@ -143,8 +143,8 @@ class TapeTest(test.TestCase): tf_rr = 2 * math_ops.reduce_sum(tf_mm) tf_da, tf_db = gradients_impl.gradients(tf_rr, [tf_a, tf_b]) - self.assertAllEqual(da.numpy(), tf_da.eval()) - self.assertAllEqual(db.numpy(), tf_db.eval()) + self.assertAllEqual(da, tf_da.eval()) + self.assertAllEqual(db, tf_db.eval()) def testGcTwoOutputs(self): @@ -155,7 +155,7 @@ class TapeTest(test.TestCase): labels = constant_op.constant([0]) logits = constant_op.constant([[0.0]]) grad, = backprop.gradients_function(fn, [0])(logits, labels) - self.assertAllEqual(grad.numpy(), [[0.0]]) + self.assertAllEqual(grad, [[0.0]]) def testTfTensor(self): @@ -164,7 +164,7 @@ class TapeTest(test.TestCase): t = constant_op.constant(1.0) g, = backprop.gradients_function(fn, [0])(t) - self.assertEqual(g.numpy(), 1.0) + self.assertAllEqual(g, 1.0) def testTapeGC(self): # TODO(apassos) figure out how to test this without using tape internal diff --git a/tensorflow/python/eager/tensor_test.py b/tensorflow/python/eager/tensor_test.py index 953807fc2a05f13d5b50cc277223e216e63237a8..2b7b5c727a2b246209629e3d293d2364b7706235 100644 --- a/tensorflow/python/eager/tensor_test.py +++ b/tensorflow/python/eager/tensor_test.py @@ -18,11 +18,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import copy + import numpy as np from tensorflow.python.eager import context from tensorflow.python.eager import core from tensorflow.python.eager import test +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util @@ -45,7 +48,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testScalarTensor(self): t = _create_tensor(3, dtype=dtypes.int32) - self.assertEqual(t.numpy(), _create_tensor(np.array(3)).numpy()) + self.assertAllEqual(t, _create_tensor(np.array(3))) self.assertEqual(dtypes.int32, t.dtype) self.assertEqual(0, t.shape.ndims) self.assertAllEqual([], t.shape.as_list()) @@ -85,12 +88,12 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testNumpyValue(self): values = np.array([3.0]) t = _create_tensor(values) - self.assertAllEqual(values, t.numpy()) + self.assertAllEqual(values, t) def testNumpyValueWithCast(self): values = np.array([3.0], dtype=np.float32) t = _create_tensor(values, dtype=dtypes.float64) - self.assertAllEqual(values, t.numpy()) + self.assertAllEqual(values, t) ctx = context.context() # Bad dtype value. with self.assertRaisesRegexp(TypeError, "Invalid dtype argument value"): @@ -100,13 +103,27 @@ class TFETensorTest(test_util.TensorFlowTestCase): def testNumpyOrderHandling(self): n = np.array([[1, 2], [3, 4]], order="F") t = _create_tensor(n) - self.assertAllEqual([[1, 2], [3, 4]], t.numpy()) + self.assertAllEqual([[1, 2], [3, 4]], t) + + def testCopy(self): + t = constant_op.constant(1.0) + tt = copy.copy(t) + self.assertAllEqual(tt, 1.0) + del tt + tt = copy.deepcopy(t) + self.assertAllEqual(tt, 1.0) + del tt + self.assertAllEqual(t, 1.0) + + def testConstantDtype(self): + self.assertEqual(constant_op.constant(1.0, dtype=np.int64).dtype, + dtypes.int64) def testTensorAndNumpyMatrix(self): expected = np.array([[1.0, 2.0], [3.0, 4.0]], np.float32) actual = _create_tensor([[1.0, 2.0], [3.0, 4.0]]) - self.assertAllEqual(expected, actual.numpy()) - self.assertEqual(np.float32, actual.numpy().dtype) + self.assertAllEqual(expected, actual) + self.assertEqual(np.float32, actual.dtype) self.assertEqual(dtypes.float32, actual.dtype) self.assertAllEqual([2, 2], actual.shape.as_list()) @@ -140,7 +157,7 @@ class TFETensorTest(test_util.TensorFlowTestCase): t = _create_tensor(np.eye(3)) tensor_str = str(t) self.assertIn("shape=%s, dtype=%s" % (t.shape, t.dtype.name), tensor_str) - self.assertIn(str(t.numpy()), tensor_str) + self.assertIn(str(t), tensor_str) def testMultiLineTensorRepr(self): t = _create_tensor(np.eye(3)) diff --git a/tensorflow/python/eager/test.py b/tensorflow/python/eager/test.py index 3d8af7e0566309e8bd2be5b857194a281a7a616f..f6a46e7eb3d03982f07bf4162d94c6038217bf61 100644 --- a/tensorflow/python/eager/test.py +++ b/tensorflow/python/eager/test.py @@ -18,11 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.eager import context as _context +from tensorflow.python.framework import ops as _ops from tensorflow.python.platform import test as _test from tensorflow.python.platform.test import * # pylint: disable=wildcard-import def main(argv=None): - _context.enable_eager_execution() + _ops.enable_eager_execution() _test.main(argv) diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD index e4b2d95acd00beed8a1545177597f3e91d4cb854..13fbfe9f5377cc8d8b475b385217ac958a8026b4 100644 --- a/tensorflow/python/estimator/BUILD +++ b/tensorflow/python/estimator/BUILD @@ -146,6 +146,7 @@ py_test( srcs = ["training_test.py"], shard_count = 4, srcs_version = "PY2AND3", + tags = ["notsan"], deps = [ ":dnn", ":estimator", @@ -542,7 +543,6 @@ py_library( "//tensorflow/python:math_ops", "//tensorflow/python:metrics", "//tensorflow/python:nn", - "//tensorflow/python:platform", "//tensorflow/python:sparse_tensor", "//tensorflow/python:string_ops", "//tensorflow/python:summary", diff --git a/tensorflow/python/estimator/canned/dnn.py b/tensorflow/python/estimator/canned/dnn.py index a3e3756007b0815933664745337b77104eb4cc52..8e90fd4ec61c72cd12c3bb2c69c31cd465903cc7 100644 --- a/tensorflow/python/estimator/canned/dnn.py +++ b/tensorflow/python/estimator/canned/dnn.py @@ -259,6 +259,10 @@ class DNNClassifier(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, @@ -392,6 +396,10 @@ class DNNRegressor(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, diff --git a/tensorflow/python/estimator/canned/dnn_linear_combined.py b/tensorflow/python/estimator/canned/dnn_linear_combined.py index ff4ecee5c026a8e0c583fd10690d6acab2f7841d..3c61bd5b07ba04193f0ed9de3567264b898114cf 100644 --- a/tensorflow/python/estimator/canned/dnn_linear_combined.py +++ b/tensorflow/python/estimator/canned/dnn_linear_combined.py @@ -278,6 +278,10 @@ class DNNLinearCombinedClassifier(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, @@ -438,6 +442,10 @@ class DNNLinearCombinedRegressor(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py index b796a3f954ff0203f0e480549ef18d4e21b2a7f0..18806db5ebea042acb3c88403af4986be012a656 100644 --- a/tensorflow/python/estimator/canned/head.py +++ b/tensorflow/python/estimator/canned/head.py @@ -40,7 +40,6 @@ from tensorflow.python.ops import nn from tensorflow.python.ops import string_ops from tensorflow.python.ops import weights_broadcast_ops from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import tf_logging as logging from tensorflow.python.saved_model import signature_constants from tensorflow.python.summary import summary @@ -269,6 +268,21 @@ def _indicator_labels_mean(labels, weights=None, name=None): return metrics_lib.mean(labels, weights=weights, name=scope) +def _classification_output(scores, n_classes, label_vocabulary=None): + batch_size = array_ops.shape(scores)[0] + if label_vocabulary: + export_class_list = label_vocabulary + else: + export_class_list = string_ops.as_string(math_ops.range(n_classes)) + export_output_classes = array_ops.tile( + input=array_ops.expand_dims(input=export_class_list, axis=0), + multiples=[batch_size, 1]) + return export_output.ClassificationOutput( + scores=scores, + # `ClassificationOutput` requires string classes. + classes=export_output_classes) + + def _accuracy_baseline(labels_mean): """Return accuracy baseline based on labels mean. @@ -299,9 +313,6 @@ def _predictions_mean(predictions, weights=None, name=None): def _auc(labels, predictions, weights=None, curve='ROC', name=None): with ops.name_scope(name, 'auc', (predictions, labels, weights)) as scope: predictions = math_ops.to_float(predictions, name='predictions') - if labels.dtype.base_dtype != dtypes.bool: - logging.warning('Casting %s labels to bool.', labels.dtype) - labels = math_ops.cast(labels, dtypes.bool) if weights is not None: weights = weights_broadcast_ops.broadcast_weights(weights, predictions) return metrics_lib.auc( @@ -355,13 +366,13 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes, `tf.feature_column.numeric_column` defining feature column representing weights. It is used to down weight or boost examples during training. It will be multiplied by the loss of the example. - label_vocabulary: A list of strings represents possible label values. If it - is not given, that means labels are already encoded as integer within - [0, n_classes). If given, labels must be string type and have any value in - `label_vocabulary`. Also there will be errors if vocabulary is not - provided and labels are string. + label_vocabulary: A list or tuple of strings representing possible label + values. If it is not given, that means labels are already encoded as an + integer within [0, n_classes). If given, labels must be of string type and + have any value in `label_vocabulary`. Note that errors will be raised if + `label_vocabulary` is not provided but labels are strings. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for multi class classification. @@ -371,8 +382,9 @@ def _multi_class_head_with_softmax_cross_entropy_loss(n_classes, """ if label_vocabulary is not None and not isinstance(label_vocabulary, (list, tuple)): - raise ValueError('label_vocabulary should be a list. Given type: {}'.format( - type(label_vocabulary))) + raise ValueError( + 'label_vocabulary should be a list or a tuple. Given type: {}'.format( + type(label_vocabulary))) return _MultiClassHeadWithSoftmaxCrossEntropyLoss(n_classes, weight_column, label_vocabulary, name) @@ -401,12 +413,11 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): def logits_dimension(self): return self._n_classes - def _eval_metric_ops(self, labels, probabilities, logits, - class_ids, weights, unweighted_loss): + def _eval_metric_ops(self, labels, class_ids, weights, unweighted_loss): """Returns the Eval metric ops.""" with ops.name_scope( None, 'metrics', - (labels, probabilities, logits, class_ids, weights, unweighted_loss)): + (labels, class_ids, weights, unweighted_loss)): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. @@ -427,8 +438,8 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): """Converts labels to integer id space.""" if self._label_vocabulary is None: if not labels.dtype.is_integer: - raise ValueError('Labels dtype should be integer ' - 'Instead got %s.' % labels.dtype) + raise ValueError('Labels dtype should be integer. Instead got {}.'. + format(labels.dtype)) label_ids = labels else: if labels.dtype != dtypes.string: @@ -453,7 +464,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): def create_estimator_spec( self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): logits = _check_logits(logits, self.logits_dimension) # Predict. @@ -479,18 +490,9 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: - batch_size = array_ops.shape(probabilities)[0] - export_class_list = self._label_vocabulary - if not export_class_list: - export_class_list = string_ops.as_string( - math_ops.range(self._n_classes)) - export_output_classes = array_ops.tile( - input=array_ops.expand_dims(input=export_class_list, axis=0), - multiples=[batch_size, 1]) - classifier_output = export_output.ClassificationOutput( - scores=probabilities, - # `ClassificationOutput` requires string classes. - classes=export_output_classes) + classifier_output = _classification_output( + scores=probabilities, n_classes=self._n_classes, + label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, @@ -513,15 +515,13 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head): loss=training_loss, eval_metric_ops=self._eval_metric_ops( labels=label_ids, - probabilities=probabilities, - logits=logits, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) # Train. if train_op_fn is None: - raise ValueError('train_op_fn can not be None.') + raise ValueError('train_op_fn cannot be None.') with ops.name_scope(''): summary.scalar( _summary_key(self._name, metric_keys.MetricKeys.LOSS), @@ -556,13 +556,13 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss( generated for each threshold value. This threshold is applied to the logistic values to determine the binary classification (i.e., above the threshold is `true`, below is `false`. - label_vocabulary: A list of strings represents possible label values. If it - is not given, that means labels are already encoded within [0, 1]. If - given, labels must be string type and have any value in - `label_vocabulary`. Also there will be errors if vocabulary is not - provided and labels are string. + label_vocabulary: A list or tuple of strings representing possible label + values. If it is not given, that means labels are already encoded within + [0, 1]. If given, labels must be string type and have any value in + `label_vocabulary`. Note that errors will be raised if `label_vocabulary` + is not provided but labels are strings. name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `Head` for binary classification. @@ -573,12 +573,13 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss( thresholds = tuple(thresholds) if thresholds else tuple() if label_vocabulary is not None and not isinstance(label_vocabulary, (list, tuple)): - raise ValueError('label_vocabulary should be a list. Given type: {}'.format( - type(label_vocabulary))) + raise ValueError( + 'label_vocabulary should be a list or tuple. Given type: {}'.format( + type(label_vocabulary))) for threshold in thresholds: if (threshold <= 0.0) or (threshold >= 1.0): - raise ValueError('thresholds not in (0, 1): %s.' % (thresholds,)) + raise ValueError('thresholds not in (0, 1): {}.'.format((thresholds,))) return _BinaryLogisticHeadWithSigmoidCrossEntropyLoss( weight_column=weight_column, thresholds=thresholds, @@ -611,12 +612,12 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): labels, logits, logistic, - scores, class_ids, unweighted_loss, weights=None): - with ops.name_scope(None, 'metrics', (labels, logits, logistic, scores, - class_ids, unweighted_loss, weights)): + with ops.name_scope( + None, 'metrics', + (labels, logits, logistic, class_ids, unweighted_loss, weights)): keys = metric_keys.MetricKeys labels_mean = _indicator_labels_mean( labels=labels, weights=weights, name=keys.LABEL_MEAN) @@ -702,14 +703,15 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): with ops.name_scope(None, 'predictions', (logits,)): pred_keys = prediction_keys.PredictionKeys logits = _check_logits(logits, self.logits_dimension) logistic = math_ops.sigmoid(logits, name=pred_keys.LOGISTIC) two_class_logits = array_ops.concat( (array_ops.zeros_like(logits), logits), 1, name='two_class_logits') - scores = nn.softmax(two_class_logits, name=pred_keys.PROBABILITIES) + probabilities = nn.softmax( + two_class_logits, name=pred_keys.PROBABILITIES) class_ids = array_ops.reshape( math_ops.argmax(two_class_logits, axis=1), (-1, 1), name='classes') if self._label_vocabulary: @@ -722,22 +724,14 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): predictions = { pred_keys.LOGITS: logits, pred_keys.LOGISTIC: logistic, - pred_keys.PROBABILITIES: scores, + pred_keys.PROBABILITIES: probabilities, pred_keys.CLASS_IDS: class_ids, pred_keys.CLASSES: classes, } if mode == model_fn.ModeKeys.PREDICT: - batch_size = array_ops.shape(logistic)[0] - export_class_list = self._label_vocabulary - if not export_class_list: - export_class_list = string_ops.as_string([0, 1]) - export_output_classes = array_ops.tile( - input=array_ops.expand_dims(input=export_class_list, axis=0), - multiples=[batch_size, 1]) - classifier_output = export_output.ClassificationOutput( - scores=scores, - # `ClassificationOutput` requires string classes. - classes=export_output_classes) + classifier_output = _classification_output( + scores=probabilities, n_classes=2, + label_vocabulary=self._label_vocabulary) return model_fn.EstimatorSpec( mode=model_fn.ModeKeys.PREDICT, predictions=predictions, @@ -764,7 +758,6 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head): labels=processed_labels, logits=logits, logistic=logistic, - scores=scores, class_ids=class_ids, unweighted_loss=unweighted_loss, weights=weights)) @@ -802,7 +795,7 @@ def _regression_head_with_mean_squared_error_loss(weight_column=None, of the last dimension of the labels `Tensor` (typically, this has shape `[batch_size, label_dimension]`). name: name of the head. If provided, summary and metrics keys will be - suffixed by `"/" + name`. + suffixed by `"/" + name`. Also used as `name_scope` when creating ops. Returns: An instance of `_Head` for linear regression. @@ -846,7 +839,7 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head): self, features, mode, logits, labels=None, train_op_fn=None): """See `Head`.""" # Predict. - with ops.name_scope('head'): + with ops.name_scope(self._name, 'head'): logits = _check_logits(logits, self._logits_dimension) predictions = {prediction_keys.PredictionKeys.PREDICTIONS: logits} if mode == model_fn.ModeKeys.PREDICT: diff --git a/tensorflow/python/estimator/canned/linear.py b/tensorflow/python/estimator/canned/linear.py index 3338f8ee2c66fff9c413caf4715b064939cdd0e8..8658ee38e99a5a6ba16560774302a1d6de8bc49e 100644 --- a/tensorflow/python/estimator/canned/linear.py +++ b/tensorflow/python/estimator/canned/linear.py @@ -184,6 +184,10 @@ class LinearClassifier(estimator.Estimator): whose `value` is a `Tensor`. Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, @@ -300,6 +304,10 @@ class LinearRegressor(estimator.Estimator): key=column.name, value=a `Tensor` Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py index 4dfc53aadfb8adbe91c422c635f420fcc35b8c0a..6243cfc118b6cc16cb6d6bfeb9ad5aab72a6d702 100644 --- a/tensorflow/python/estimator/estimator.py +++ b/tensorflow/python/estimator/estimator.py @@ -29,6 +29,7 @@ import six from tensorflow.core.framework import summary_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as tf_session +from tensorflow.python.eager import context from tensorflow.python.estimator import model_fn as model_fn_lib from tensorflow.python.estimator import run_config from tensorflow.python.estimator import util @@ -52,7 +53,6 @@ from tensorflow.python.training import training from tensorflow.python.training import training_util from tensorflow.python.util import compat from tensorflow.python.util import nest -from tensorflow.python.util import tf_inspect _VALID_MODEL_FN_ARGS = set( @@ -88,6 +88,10 @@ class Estimator(object): None of `Estimator`'s methods can be overridden in subclasses (its constructor enforces this). Subclasses should use `model_fn` to configure the base class, and may add methods implementing specialized functionality. + + @compatibility(eager) + Estimators are not compatible with eager execution. + @end_compatibility """ def __init__(self, model_fn, model_dir=None, config=None, params=None): @@ -130,10 +134,15 @@ class Estimator(object): Keys are names of parameters, values are basic python types. Raises: + RuntimeError: If eager execution is enabled. ValueError: parameters of `model_fn` don't match `params`. ValueError: if this is called via a subclass and if that class overrides a member of `Estimator`. """ + if context.in_eager_mode(): + raise RuntimeError( + 'Estimators are not supported when eager execution is enabled.') + Estimator._assert_members_are_not_overridden(self) if config is None: @@ -707,10 +716,9 @@ class Estimator(object): with ops.Graph().as_default() as g, g.device(self._device_fn): random_seed.set_random_seed(self._config.tf_random_seed) global_step_tensor = self._create_and_assert_global_step(g) - global_step_read_tensor = training_util._get_or_create_global_step_read() # pylint: disable=protected-access - with ops.control_dependencies([global_step_read_tensor]): - features, labels = self._get_features_and_labels_from_input_fn( - input_fn, model_fn_lib.ModeKeys.TRAIN) + training_util._get_or_create_global_step_read() # pylint: disable=protected-access + features, labels = self._get_features_and_labels_from_input_fn( + input_fn, model_fn_lib.ModeKeys.TRAIN) estimator_spec = self._call_model_fn( features, labels, model_fn_lib.ModeKeys.TRAIN, self.config) # Check if the user created a loss summary, and add one if they didn't. @@ -926,9 +934,6 @@ def _verify_model_fn_args(model_fn, params): logging.warning('Estimator\'s model_fn (%s) includes params ' 'argument, but params are not passed to Estimator.', model_fn) - if tf_inspect.ismethod(model_fn): - if 'self' in args: - args.remove('self') non_valid_args = list(args - _VALID_MODEL_FN_ARGS) if non_valid_args: raise ValueError('model_fn (%s) has following not expected args: %s' % @@ -1021,4 +1026,3 @@ def _has_dataset_or_queue_runner(maybe_tensor): # Now, check queue. return ops.get_default_graph().get_collection(ops.GraphKeys.QUEUE_RUNNERS) - diff --git a/tensorflow/python/estimator/export/export.py b/tensorflow/python/estimator/export/export.py index e2e20f0d717b7dec7a968222b2d76f315b2b538f..31e9933c6f702393eb21b10c5bdd770739056032 100644 --- a/tensorflow/python/estimator/export/export.py +++ b/tensorflow/python/estimator/export/export.py @@ -33,6 +33,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import gfile from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.util import compat @@ -47,8 +48,8 @@ class ServingInputReceiver(collections.namedtuple( """A return type for a serving_input_receiver_fn. The expected return values are: - features: A dict of string to `Tensor` or `SparseTensor`, specifying the - features to be passed to the model. + features: A `Tensor`, `SparseTensor`, or dict of string to `Tensor` or + `SparseTensor`, specifying the features to be passed to the model. receiver_tensors: a `Tensor`, or a dict of string to `Tensor`, specifying input nodes where this receiver expects to be fed by default. Typically, this is a single placeholder expecting serialized `tf.Example` protos. @@ -193,13 +194,14 @@ def build_all_signature_defs(receiver_tensors, raise ValueError('export_outputs must be a dict.') signature_def_map = {} + excluded_signatures = {} for output_key, export_output in export_outputs.items(): signature_name = '{}'.format(output_key or 'None') try: signature = export_output.as_signature_def(receiver_tensors) signature_def_map[signature_name] = signature - except ValueError: - pass + except ValueError as e: + excluded_signatures[signature_name] = str(e) if receiver_tensors_alternatives: for receiver_name, receiver_tensors_alt in ( @@ -213,8 +215,10 @@ def build_all_signature_defs(receiver_tensors, try: signature = export_output.as_signature_def(receiver_tensors_alt) signature_def_map[signature_name] = signature - except ValueError: - pass + except ValueError as e: + excluded_signatures[signature_name] = str(e) + + _log_signature_report(signature_def_map, excluded_signatures) # The above calls to export_output.as_signature_def should return only # valid signatures; if there is a validity problem, they raise ValueError, @@ -224,6 +228,46 @@ def build_all_signature_defs(receiver_tensors, if signature_def_utils.is_valid_signature(v)} +_FRIENDLY_METHOD_NAMES = { + signature_constants.CLASSIFY_METHOD_NAME: 'Classify', + signature_constants.REGRESS_METHOD_NAME: 'Regress', + signature_constants.PREDICT_METHOD_NAME: 'Predict', +} + + +def _log_signature_report(signature_def_map, excluded_signatures): + """Log a report of which signatures were produced.""" + sig_names_by_method_name = collections.defaultdict(list) + + # We'll collect whatever method_names are present, but also we want to make + # sure to output a line for each of the three standard methods even if they + # have no signatures. + for method_name in _FRIENDLY_METHOD_NAMES: + sig_names_by_method_name[method_name] = [] + + for signature_name, sig in signature_def_map.items(): + sig_names_by_method_name[sig.method_name].append(signature_name) + + # TODO(b/67733540): consider printing the full signatures, not just names + for method_name, sig_names in sig_names_by_method_name.items(): + if method_name in _FRIENDLY_METHOD_NAMES: + method_name = _FRIENDLY_METHOD_NAMES[method_name] + logging.info('Signatures INCLUDED in export for {}: {}'.format( + method_name, sig_names if sig_names else 'None')) + + if excluded_signatures: + logging.info('Signatures EXCLUDED from export because they cannot be ' + 'be served via TensorFlow Serving APIs:') + for signature_name, message in excluded_signatures.items(): + logging.info('\'{}\' : {}'.format(signature_name, message)) + + if not signature_def_map: + logging.warn('Export includes no signatures!') + elif (signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY + not in signature_def_map): + logging.warn('Export includes no default signature!') + + # When we create a timestamped directory, there is a small chance that the # directory already exists because another worker is also writing exports. # In this case we just wait one second to get a new timestamp and try again. diff --git a/tensorflow/python/estimator/export/export_output.py b/tensorflow/python/estimator/export/export_output.py index 7c7f92872ebb10c9679c07aa3bb15bfbf5021b4d..863af6d41d985043542b03375372fe564c283b82 100644 --- a/tensorflow/python/estimator/export/export_output.py +++ b/tensorflow/python/estimator/export/export_output.py @@ -150,6 +150,9 @@ class RegressionOutput(ExportOutput): return signature_def_utils.regression_signature_def(examples, self.value) +_SINGLE_OUTPUT_DEFAULT_NAME = 'output' + + class PredictOutput(ExportOutput): """Represents the output of a generic prediction head. @@ -162,16 +165,15 @@ class PredictOutput(ExportOutput): """Constructor for PredictOutput. Args: - outputs: A dict of string to `Tensor` representing the predictions. + outputs: A `Tensor` or a dict of string to `Tensor` representing the + predictions. Raises: ValueError: if the outputs is not dict, or any of its keys are not strings, or any of its values are not `Tensor`s. """ if not isinstance(outputs, dict): - raise ValueError( - 'Prediction outputs must be given as a dict of string to Tensor; ' - 'got {}'.format(outputs)) + outputs = {_SINGLE_OUTPUT_DEFAULT_NAME: outputs} for key, value in outputs.items(): if not isinstance(key, six.string_types): raise ValueError( diff --git a/tensorflow/python/estimator/export/export_output_test.py b/tensorflow/python/estimator/export/export_output_test.py index 035a9a143e6ffa18ae78ef2544614f342363b22d..7090e53d807817db7d66ed0ee1307d7e38e9e87e 100644 --- a/tensorflow/python/estimator/export/export_output_test.py +++ b/tensorflow/python/estimator/export/export_output_test.py @@ -199,20 +199,18 @@ class ExportOutputTest(test.TestCase): signature_constants.CLASSIFY_METHOD_NAME) self.assertEqual(actual_signature_def, expected_signature_def) - def test_predict_output_constructor(self): - """Tests that no errors are raised when input is expected.""" + def test_predict_outputs_valid(self): + """Tests that no errors are raised when provided outputs are valid.""" outputs = { "output0": constant_op.constant([0]), - u"output1": constant_op.constant([1]), + u"output1": constant_op.constant(["foo"]), } export_output_lib.PredictOutput(outputs) - def test_predict_output_outputs_invalid(self): - with self.assertRaisesRegexp( - ValueError, - "Prediction outputs must be given as a dict of string to Tensor"): - export_output_lib.PredictOutput(constant_op.constant([0])) + # Single Tensor is OK too + export_output_lib.PredictOutput(constant_op.constant([0])) + def test_predict_outputs_invalid(self): with self.assertRaisesRegexp( ValueError, "Prediction output key must be a string"): diff --git a/tensorflow/python/estimator/exporter.py b/tensorflow/python/estimator/exporter.py index 56400ab935a9f44baea16e19feb9db5a6c6f581c..c6f20d4a9e2a6b3384ba59ae2df67ff7a3464aa9 100644 --- a/tensorflow/python/estimator/exporter.py +++ b/tensorflow/python/estimator/exporter.py @@ -71,7 +71,7 @@ class _SavedModelExporter(Exporter): def __init__(self, name, - serving_input_fn, + serving_input_receiver_fn, assets_extra=None, as_text=False): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -79,8 +79,8 @@ class _SavedModelExporter(Exporter): Args: name: unique name of this `Exporter` that is going to be used in the export path. - serving_input_fn: a function that takes no arguments and returns an - `ServingInputReceiver`. + serving_input_receiver_fn: a function that takes no arguments and returns + a `ServingInputReceiver`. assets_extra: An optional dict specifying how to populate the assets.extra directory within the exported SavedModel. Each key should give the destination path (including the filename) relative to the assets.extra @@ -95,7 +95,7 @@ class _SavedModelExporter(Exporter): ValueError: if any arguments is invalid. """ self._name = name - self._serving_input_fn = serving_input_fn + self._serving_input_receiver_fn = serving_input_receiver_fn self._assets_extra = assets_extra self._as_text = as_text @@ -109,7 +109,7 @@ class _SavedModelExporter(Exporter): export_result = estimator.export_savedmodel( export_path, - self._serving_input_fn, + self._serving_input_receiver_fn, assets_extra=self._assets_extra, as_text=self._as_text, checkpoint_path=checkpoint_path) @@ -125,7 +125,7 @@ class FinalExporter(Exporter): def __init__(self, name, - serving_input_fn, + serving_input_receiver_fn, assets_extra=None, as_text=False): """Create an `Exporter` to use with `tf.estimator.EvalSpec`. @@ -133,8 +133,8 @@ class FinalExporter(Exporter): Args: name: unique name of this `Exporter` that is going to be used in the export path. - serving_input_fn: a function that takes no arguments and returns an - `ServingInputReceiver`. + serving_input_receiver_fn: a function that takes no arguments and returns + a `ServingInputReceiver`. assets_extra: An optional dict specifying how to populate the assets.extra directory within the exported SavedModel. Each key should give the destination path (including the filename) relative to the assets.extra @@ -148,7 +148,8 @@ class FinalExporter(Exporter): Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn, + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, assets_extra, as_text) @property @@ -175,7 +176,7 @@ class LatestExporter(Exporter): def __init__(self, name, - serving_input_fn, + serving_input_receiver_fn, assets_extra=None, as_text=False, exports_to_keep=5): @@ -184,8 +185,8 @@ class LatestExporter(Exporter): Args: name: unique name of this `Exporter` that is going to be used in the export path. - serving_input_fn: a function that takes no arguments and returns an - `ServingInputReceiver`. + serving_input_receiver_fn: a function that takes no arguments and returns + a `ServingInputReceiver`. assets_extra: An optional dict specifying how to populate the assets.extra directory within the exported SavedModel. Each key should give the destination path (including the filename) relative to the assets.extra @@ -202,7 +203,8 @@ class LatestExporter(Exporter): Raises: ValueError: if any arguments is invalid. """ - self._saved_model_exporter = _SavedModelExporter(name, serving_input_fn, + self._saved_model_exporter = _SavedModelExporter(name, + serving_input_receiver_fn, assets_extra, as_text) self._exports_to_keep = exports_to_keep if exports_to_keep is not None and exports_to_keep <= 0: diff --git a/tensorflow/python/estimator/exporter_test.py b/tensorflow/python/estimator/exporter_test.py index f90c35dce72acd965ea5868361e18304402e4e6d..8e0f66cece754dea95987d136d90855e6818236b 100644 --- a/tensorflow/python/estimator/exporter_test.py +++ b/tensorflow/python/estimator/exporter_test.py @@ -33,19 +33,19 @@ from tensorflow.python.util import compat class LatestExporterTest(test.TestCase): def test_error_out_if_exports_to_keep_is_zero(self): - def _serving_input_fn(): + def _serving_input_receiver_fn(): pass with self.assertRaisesRegexp(ValueError, "positive number"): exporter = exporter_lib.LatestExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, exports_to_keep=0) self.assertEqual("latest_exporter", exporter.name) def test_latest_exporter(self): - def _serving_input_fn(): + def _serving_input_receiver_fn(): pass export_dir_base = tempfile.mkdtemp() + "export/" @@ -53,7 +53,7 @@ class LatestExporterTest(test.TestCase): exporter = exporter_lib.LatestExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False, exports_to_keep=5) @@ -66,14 +66,14 @@ class LatestExporterTest(test.TestCase): self.assertEqual("export_result_path", export_result) estimator.export_savedmodel.assert_called_with( export_dir_base, - _serving_input_fn, + _serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False, checkpoint_path="checkpoint_path") def test_only_the_last_export_is_saved(self): - def _serving_input_fn(): + def _serving_input_receiver_fn(): pass export_dir_base = tempfile.mkdtemp() + "export/" @@ -81,7 +81,7 @@ class LatestExporterTest(test.TestCase): exporter = exporter_lib.FinalExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False) estimator = test.mock.Mock(spec=estimator_lib.Estimator) @@ -99,7 +99,7 @@ class LatestExporterTest(test.TestCase): self.assertEqual("export_result_path", export_result) estimator.export_savedmodel.assert_called_with( export_dir_base, - _serving_input_fn, + _serving_input_receiver_fn, assets_extra={"from/path": "to/path"}, as_text=False, checkpoint_path="checkpoint_path") @@ -117,12 +117,12 @@ class LatestExporterTest(test.TestCase): self.assertTrue(gfile.Exists(export_dir_3)) self.assertTrue(gfile.Exists(export_dir_4)) - def _serving_input_fn(): + def _serving_input_receiver_fn(): return array_ops.constant([1]), None exporter = exporter_lib.LatestExporter( name="latest_exporter", - serving_input_fn=_serving_input_fn, + serving_input_receiver_fn=_serving_input_receiver_fn, exports_to_keep=2) estimator = test.mock.Mock(spec=estimator_lib.Estimator) # Garbage collect all but the most recent 2 exports, diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py index 1820b2b2d4cdc0422f4c2a42be67efb83286fc0c..d71964d2ec8e8ce21934428c3fff88f65b2751da 100644 --- a/tensorflow/python/estimator/run_config.py +++ b/tensorflow/python/estimator/run_config.py @@ -210,7 +210,7 @@ class RunConfig(object): def __init__(self, model_dir=None, - tf_random_seed=1, + tf_random_seed=None, save_summary_steps=100, save_checkpoints_steps=_USE_DEFAULT, save_checkpoints_secs=_USE_DEFAULT, @@ -528,6 +528,7 @@ class RunConfig(object): """Returns a new instance of `RunConfig` replacing specified properties. Only the properties in the following list are allowed to be replaced: + - `model_dir`. - `tf_random_seed`, - `save_summary_steps`, diff --git a/tensorflow/python/estimator/run_config_test.py b/tensorflow/python/estimator/run_config_test.py index b3c917649f4712f492f713b13e8847e5a784db41..ecc850d5405837e8bf803b9a7c8c156ff19b7a90 100644 --- a/tensorflow/python/estimator/run_config_test.py +++ b/tensorflow/python/estimator/run_config_test.py @@ -70,7 +70,7 @@ class RunConfigTest(test.TestCase): config = run_config_lib.RunConfig() self.assertIsNone(config.model_dir) self.assertIsNone(config.session_config) - self.assertEqual(1, config.tf_random_seed) + self.assertIsNone(config.tf_random_seed) self.assertEqual(100, config.save_summary_steps) self.assertEqual(600, config.save_checkpoints_secs) self.assertIsNone(config.save_checkpoints_steps) diff --git a/tensorflow/python/estimator/training_test.py b/tensorflow/python/estimator/training_test.py index d88ca2c925c7544dd1e73b4310d486c3a2f847fe..1862e325e2b65ae2141132c4b900673c755e179e 100644 --- a/tensorflow/python/estimator/training_test.py +++ b/tensorflow/python/estimator/training_test.py @@ -1569,7 +1569,7 @@ class TrainAndEvaluateIntegrationTest(test.TestCase): serving_input_receiver_fn = ( export_lib.build_parsing_serving_input_receiver_fn(feature_spec)) return exporter_lib.LatestExporter( - name, serving_input_fn=serving_input_receiver_fn) + name, serving_input_receiver_fn=serving_input_receiver_fn) def _extract_loss_and_global_step(self, event_folder): """Returns the loss and global step in last event.""" diff --git a/tensorflow/python/estimator/util.py b/tensorflow/python/estimator/util.py index de35e66bdfb46dbfdc0be3b4316d62a3a136142a..12f2592d848c3ce55777ffdae5cee7ac602ee87f 100644 --- a/tensorflow/python/estimator/util.py +++ b/tensorflow/python/estimator/util.py @@ -19,10 +19,20 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + from tensorflow.python.util import tf_decorator from tensorflow.python.util import tf_inspect +def _is_bounded_method(fn): + return tf_inspect.ismethod(fn) and (fn.__self__ is not None) + + +def _is_callable_object(obj): + return hasattr(obj, '__call__') and tf_inspect.ismethod(obj.__call__) + + def fn_args(fn): """Get argument names for function-like object. @@ -36,22 +46,13 @@ def fn_args(fn): ValueError: if partial function has positionally bound arguments """ _, fn = tf_decorator.unwrap(fn) - - # Handle callables. - if hasattr(fn, '__call__') and tf_inspect.ismethod(fn.__call__): - return tuple(tf_inspect.getargspec(fn.__call__).args) - - # Handle functools.partial and similar objects. - if hasattr(fn, 'func') and hasattr(fn, 'keywords') and hasattr(fn, 'args'): - # Handle nested partial. - original_args = fn_args(fn.func) - if not original_args: - return tuple() - - return tuple([ - arg for arg in original_args[len(fn.args):] - if arg not in set((fn.keywords or {}).keys()) - ]) - - # Handle function. - return tuple(tf_inspect.getargspec(fn).args) + if isinstance(fn, functools.partial): + args = fn_args(fn.func) + args = [a for a in args[len(fn.args):] if a not in (fn.keywords or [])] + else: + if _is_callable_object(fn): + fn = fn.__call__ + args = tf_inspect.getargspec(fn).args + if _is_bounded_method(fn): + args.remove('self') + return tuple(args) diff --git a/tensorflow/python/estimator/util_test.py b/tensorflow/python/estimator/util_test.py index 3f8122c407bfc707d1d411ca6ed31b6ad72ee6a2..4b2c8d7637e2e67369e8e2a679c328c10791a52e 100644 --- a/tensorflow/python/estimator/util_test.py +++ b/tensorflow/python/estimator/util_test.py @@ -38,7 +38,16 @@ class FnArgsTest(test.TestCase): def __call__(self, a, b): return a + b - self.assertEqual(('self', 'a', 'b'), util.fn_args(Foo())) + self.assertEqual(('a', 'b'), util.fn_args(Foo())) + + def test_bounded_method(self): + + class Foo(object): + + def bar(self, a, b): + return a + b + + self.assertEqual(('a', 'b'), util.fn_args(Foo().bar)) def test_partial_function(self): expected_test_arg = 123 diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD index 27062adb611a3f270f402d664e3708ff5c66918b..b1c81dd58c7d2d9cf95821ea78eda2e7ee675d25 100644 --- a/tensorflow/python/feature_column/BUILD +++ b/tensorflow/python/feature_column/BUILD @@ -86,6 +86,7 @@ py_test( "//tensorflow/python:framework_ops", "//tensorflow/python:lookup_ops", "//tensorflow/python:parsing_ops", + "//tensorflow/python:partitioned_variables", "//tensorflow/python:session", "//tensorflow/python:sparse_tensor", "//tensorflow/python:training", diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py index 81f4f45fcbc71eeaa57686f791cf4d1621ba9ce7..190a25d4d79e9acc1986f5bd06110a29f29aee42 100644 --- a/tensorflow/python/feature_column/feature_column.py +++ b/tensorflow/python/feature_column/feature_column.py @@ -197,12 +197,13 @@ def input_layer(features, trainable: If `True` also add the variable to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). cols_to_vars: If not `None`, must be a dictionary that will be filled with a - mapping from `_FeatureColumn` to associated `Variable` (or list of - `Variable`, or `PartitionedVariable`. For example, after the call, we - might have cols_to_vars = {_EmbeddingColumn( + mapping from `_FeatureColumn` to list of `Variable`s. For example, after + the call, we might have cols_to_vars = + {_EmbeddingColumn( categorical_column=_HashedCategoricalColumn( key='sparse_feature', hash_bucket_size=5, dtype=tf.string), - dimension=10): [, - 'bias': , + [], + 'bias': [], _NumericColumn( key='numeric_feature2', shape=(2,)): - } - Note that it will also contain a string key 'bias'. If a column creates - no variables, its value will be an empty list. + []} + If a column creates no variables, its value will be an empty list. Note + that cols_to_vars will also contain a string key 'bias' that maps to a + list of Variables. Returns: A `Tensor` which represents predictions/logits of a linear model. Its shape @@ -366,8 +367,12 @@ def linear_model(features, predictions = nn_ops.bias_add( predictions_no_bias, bias, name='weighted_sum') if cols_to_vars is not None: - # Add the bias to cols_to_vars as well. - cols_to_vars['bias'] = bias + # Add the bias to cols_to_vars as well, converting the Variable or + # PartitionedVariable to a list of Variable's. + if isinstance(bias, variables.Variable): + cols_to_vars['bias'] = [bias] + else: # Must be a PartitionedVariable. + cols_to_vars['bias'] = list(bias) return predictions diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 112600439b55b3ebd5c7cd1a40bf0fc6c3eeeb4e..e57e9a9836c1cb38b2e3cea8a9d16283049e9c7d 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -41,6 +41,7 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import parsing_ops +from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables as variables_lib from tensorflow.python.platform import test @@ -1354,10 +1355,33 @@ class LinearModelTest(test.TestCase): bias = get_linear_model_bias() price1_var = get_linear_model_column_var(price1) price2_var = get_linear_model_column_var(price2) - self.assertEqual(cols_to_vars['bias'], bias) + self.assertAllEqual(cols_to_vars['bias'], [bias]) self.assertAllEqual(cols_to_vars[price1], [price1_var]) self.assertAllEqual(cols_to_vars[price2], [price2_var]) + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1', shape=2) + price2 = fc.numeric_column('price2', shape=3) + with ops.Graph().as_default(): + features = { + 'price1': [[1., 2.], [6., 7.]], + 'price2': [[3., 4., 5.], [8., 9., 10.]] + } + cols_to_vars = {} + with variable_scope.variable_scope( + 'linear', + partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)): + fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars) + with _initialized_session(): + self.assertEqual([0.], cols_to_vars['bias'][0].eval()) + # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables. + self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval()) + # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and + # a [1, 1] Variable. + self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval()) + self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval()) + def test_dense_collection(self): price = fc.numeric_column('price') with ops.Graph().as_default() as g: @@ -1761,9 +1785,38 @@ class InputLayerTest(test.TestCase): self.assertEqual(0, len(cols_to_vars[price1])) self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized])) self.assertEqual(1, len(cols_to_vars[some_embedding_column])) - for var in cols_to_vars[some_embedding_column]: - self.assertIsInstance(var, variables_lib.Variable) - self.assertAllEqual(var.shape, [5, 10]) + self.assertIsInstance(cols_to_vars[some_embedding_column][0], + variables_lib.Variable) + self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [5, 10]) + + def test_fills_cols_to_vars_partitioned_variables(self): + price1 = fc.numeric_column('price1') + dense_feature = fc.numeric_column('dense_feature') + dense_feature_bucketized = fc.bucketized_column( + dense_feature, boundaries=[0.]) + some_sparse_column = fc.categorical_column_with_hash_bucket( + 'sparse_feature', hash_bucket_size=5) + some_embedding_column = fc.embedding_column( + some_sparse_column, dimension=10) + with ops.Graph().as_default(): + features = { + 'price1': [[3.], [4.]], + 'dense_feature': [[-1.], [4.]], + 'sparse_feature': [['a'], ['x']], + } + cols_to_vars = {} + all_cols = [price1, dense_feature_bucketized, some_embedding_column] + with variable_scope.variable_scope( + 'input_from_feature_columns', + partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0)): + fc.input_layer(features, all_cols, cols_to_vars=cols_to_vars) + self.assertItemsEqual(list(cols_to_vars.keys()), all_cols) + self.assertEqual(0, len(cols_to_vars[price1])) + self.assertEqual(0, len(cols_to_vars[dense_feature_bucketized])) + self.assertEqual(3, len(cols_to_vars[some_embedding_column])) + self.assertAllEqual(cols_to_vars[some_embedding_column][0].shape, [2, 10]) + self.assertAllEqual(cols_to_vars[some_embedding_column][1].shape, [2, 10]) + self.assertAllEqual(cols_to_vars[some_embedding_column][2].shape, [1, 10]) def test_column_order(self): price_a = fc.numeric_column('price_a') diff --git a/tensorflow/python/framework/constant_op.py b/tensorflow/python/framework/constant_op.py index 686f5aa6db41e1bfbec4c162cb2783b8cfe8c475..d51e142da1950d48eaa38ebc2366da6912cb19e7 100644 --- a/tensorflow/python/framework/constant_op.py +++ b/tensorflow/python/framework/constant_op.py @@ -108,7 +108,10 @@ def convert_to_eager_tensor(value, ctx, dtype=None): dtype, value.dtype)) return value if dtype is not None: - dtype = dtype.as_datatype_enum + try: + dtype = dtype.as_datatype_enum + except AttributeError: + dtype = dtypes.as_dtype(dtype).as_datatype_enum device = ctx.device_name handle = ctx._handle # pylint: disable=protected-access if isinstance(value, (float,) + six.integer_types): @@ -195,7 +198,7 @@ def constant(value, dtype=None, shape=None, name="Const", verify_shape=False): # We don't have a Fill kernel for bool dtype on GPU. So we first run # Fill on CPU and then copy to GPU if needed. with ops.device("/device:CPU:0"): - x = _eager_fill(shape.as_list(), t.as_cpu_tensor(), ctx) + x = _eager_fill(shape.as_list(), t.cpu(), ctx) return _eager_identity(x, ctx) else: return _eager_fill(shape.as_list(), t, ctx) diff --git a/tensorflow/python/framework/fast_tensor_util.pyx b/tensorflow/python/framework/fast_tensor_util.pyx index b43ddb4ad3af3996906370ac20643742bc105576..19928314efe143572b36324230f8e8f2ae87648d 100644 --- a/tensorflow/python/framework/fast_tensor_util.pyx +++ b/tensorflow/python/framework/fast_tensor_util.pyx @@ -30,6 +30,12 @@ def AppendInt32ArrayToTensorProto( for i in range(n): tensor_proto.int_val.append(nparray[i]) +def AppendUInt32ArrayToTensorProto( + tensor_proto, np.ndarray[np.uint32_t, ndim=1] nparray): + cdef long i, n + n = nparray.size + for i in range(n): + tensor_proto.uint32_val.append(nparray[i]) def AppendInt64ArrayToTensorProto( tensor_proto, np.ndarray[np.int64_t, ndim=1] nparray): @@ -38,6 +44,12 @@ def AppendInt64ArrayToTensorProto( for i in range(n): tensor_proto.int64_val.append(nparray[i]) +def AppendUInt64ArrayToTensorProto( + tensor_proto, np.ndarray[np.uint64_t, ndim=1] nparray): + cdef long i, n + n = nparray.size + for i in range(n): + tensor_proto.uint64_val.append(nparray[i]) def AppendUInt8ArrayToTensorProto( tensor_proto, np.ndarray[np.uint8_t, ndim=1] nparray): diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py index fea2129922cb957469ce6c36d38ea171980cb93a..36b0737cfca181a1d2c2fe6df2460312ed25dfa5 100644 --- a/tensorflow/python/framework/function_test.py +++ b/tensorflow/python/framework/function_test.py @@ -309,8 +309,7 @@ class FunctionTest(test.TestCase): self.assertAllClose(y.eval(), 6.) self.assertAllClose(dx.eval(), 2.) - def testZNoDepOnY(self): - + def _testZNoDepOnY(self, use_const_grad_ys): @function.Defun(dtypes.float32, dtypes.float32) def Foo(x, y): # pylint: disable=unused-argument return x * 2 @@ -320,12 +319,22 @@ class FunctionTest(test.TestCase): x = constant_op.constant(1.0) y = constant_op.constant(2.0) z = Foo(x, y) - dx, dy = gradients_impl.gradients([z], [x, y]) + if use_const_grad_ys: + dx, dy = gradients_impl.gradients([z], [x, y], grad_ys=[1.0]) + else: + dx, dy = gradients_impl.gradients([z], [x, y]) with session.Session() as sess: dx_val, dy_val = sess.run([dx, dy]) self.assertEqual([2.0], dx_val) self.assertEqual([0.0], dy_val) + def testZNoDepOnY(self): + self._testZNoDepOnY(False) + + def testZNoDepOnYConstGradYs(self): + # Tests for constant folding of grad_ys + self._testZNoDepOnY(True) + def testDefineFunctionNoArgs(self): @function.Defun(func_name="AConstant") @@ -855,6 +864,24 @@ class FunctionTest(test.TestCase): [result]) self.assertEqual(len(f.signature.input_arg), 3) + def testGradientWithIntegerFunctionArgument(self): + @function.Defun(dtypes.int32, dtypes.float32) + def Foo(t, x): + return x[t] + + g = ops.Graph() + with g.as_default(): + inp = array_ops.placeholder(dtypes.float32) + t = constant_op.constant(0, dtypes.int32) + out = Foo(t, inp) + dinp, = gradients_impl.gradients(out, [inp]) + + x = np.zeros((2,)).astype(np.float32) + with session.Session(graph=g) as sess: + self.assertAllClose( + np.array([1.0, 0.0]).astype(np.float32), + sess.run(dinp, {inp: x})) + @test_util.with_c_api class FunctionsFromProtos(test.TestCase): diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py index 65abb6959915e6fda9267e18e6d80577bb126cfd..06cee46bf623ff0521f4ebe91ff1909aa45e00e3 100644 --- a/tensorflow/python/framework/meta_graph_test.py +++ b/tensorflow/python/framework/meta_graph_test.py @@ -36,8 +36,10 @@ from tensorflow.python.ops import data_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import metrics from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import partitioned_variables from tensorflow.python.ops import random_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -657,5 +659,42 @@ class MetaGraphWithVariableScopeTest(test.TestCase): initializer = variables.local_variables_initializer() +class ExportImportAcrossScopesTest(test.TestCase): + + def testPartionedVariables(self): + def make_graph_with_partitioned_variables(): + variable_scope.get_variable( + name="weights", + partitioner=partitioned_variables.fixed_size_partitioner(3, axis=0), + initializer=random_ops.truncated_normal([100, 10])) + self._testExportImportAcrossScopes(make_graph_with_partitioned_variables) + + def _testExportImportAcrossScopes(self, graph_fn): + """Tests export and importing a graph across scopes. + + Args: + graph_fn: A closure that creates a graph on the current scope. + """ + with ops.Graph().as_default() as original_graph: + with variable_scope.variable_scope("dropA/dropB/keepA"): + graph_fn() + exported_meta_graph_def = meta_graph.export_scoped_meta_graph( + graph=original_graph, + export_scope="dropA/dropB")[0] + + with ops.Graph().as_default() as imported_graph: + meta_graph.import_scoped_meta_graph( + exported_meta_graph_def, + import_scope="importA") + + with ops.Graph().as_default() as expected_graph: + with variable_scope.variable_scope("importA/keepA"): + graph_fn() + + result = meta_graph.export_scoped_meta_graph(graph=imported_graph)[0] + expected = meta_graph.export_scoped_meta_graph(graph=expected_graph)[0] + self.assertProtoEquals(expected, result) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 6077d602c4085c8df6063c588b7fbc6d371ccde5..e68eac372369b25a902936467be71b6079a23ce4 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -25,6 +25,7 @@ import re import sys import threading +import numpy as np import six from six.moves import xrange # pylint: disable=redefined-builtin @@ -32,6 +33,7 @@ from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import function_pb2 from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import node_def_pb2 +from tensorflow.core.framework import op_def_pb2 from tensorflow.core.framework import versions_pb2 from tensorflow.python import pywrap_tensorflow as c_api from tensorflow.python.eager import context @@ -45,6 +47,7 @@ from tensorflow.python.framework import op_def_registry from tensorflow.python.framework import registry from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import versions +from tensorflow.python.platform import app from tensorflow.python.platform import tf_logging as logging from tensorflow.python.util import compat from tensorflow.python.util import decorator_utils @@ -380,6 +383,14 @@ class Tensor(_TensorLike): return None return tuple(shape) + def _rank(self): + """Integer rank of this Tensor, if known, else None. + + Returns: + Integer rank or None + """ + return self._shape.ndims + def get_shape(self): """Alias of Tensor.shape.""" return self.shape @@ -605,9 +616,6 @@ class _EagerTensorBase(Tensor): def numpy(self): """Returns a numpy array with the same contents as the Tensor. - The contents of the Tensor must be backed by host memory. The - as_cpu_tensor() method can be used ensure that this is true. - TODO(ashankar,agarwal): Perhaps this should NOT reference the underlying buffer but instead always explicitly copy? Note that currently it may or may not copy based on whether the numpy data is properly aligned or not. @@ -615,12 +623,37 @@ class _EagerTensorBase(Tensor): Returns: A numpy array that may share memory with the Tensor object. Any changes to one may be reflected in the other. + + Raises: + ValueError: if the type of this Tensor is not representable in numpy. """ - return self.as_cpu_tensor()._numpy() # pylint: disable=protected-access + if self.dtype == dtypes.resource: + raise ValueError("Resource handles are not convertible to numpy.") + return self.cpu()._numpy() # pylint: disable=protected-access + + # __int__ and __float__ may copy the tensor to CPU and + # only work for scalars; values are cast as per numpy. + def __int__(self): + return int(self.numpy()) + + def __float__(self): + return float(self.numpy()) + + def __array__(self): + return np.array(self.numpy()) def _numpy(self): raise NotImplementedError() + def __copy__(self): + # Eager Tensors are immutable so it's safe to return themselves as a copy. + return self + + def __deepcopy__(self, memo): + # Eager Tensors are immutable so it's safe to return themselves as a copy. + del memo + return self + def _datatype_enum(self): raise NotImplementedError() @@ -639,6 +672,18 @@ class _EagerTensorBase(Tensor): """ raise NotImplementedError() + def _rank(self): + """Integer rank of this Tensor. + + Unlike regular Tensors, the rank is always known for EagerTensors. + + This is more performant than len(self._shape_tuple()) + + Returns: + Integer rank + """ + raise NotImplementedError() + def _copy_to_device(self, context, device): # pylint: disable=redefined-outer-name raise NotImplementedError() @@ -698,11 +743,11 @@ class _EagerTensorBase(Tensor): """The shape of the tensor as a list.""" return list(self._shape_tuple()) - def as_cpu_tensor(self): + def cpu(self): """A copy of this Tensor with contents backed by host memory.""" return self._copy(context.context(), "CPU:0") - def as_gpu_tensor(self, gpu_index=0): + def gpu(self, gpu_index=0): """A copy of this Tensor with contents backed by memory on the GPU. Arguments: @@ -722,30 +767,33 @@ class _EagerTensorBase(Tensor): if self.dtype != dtypes.bool: raise ValueError( "Non-boolean tensor %s cannot be converted to boolean." % repr(self)) - return bool(self.as_cpu_tensor().numpy()) + return bool(self.cpu().numpy()) def __nonzero__(self): return self.__bool__() + def set_shape(self, shape): + if not self.shape.is_compatible_with(shape): + raise ValueError( + "EagerTensor's shape %s is not compatible with supplied shape %s" % + (self.shape, shape)) + # Methods not supported / implemented for Eager Tensors. @property def op(self): - raise NotImplementedError("op not supported for Eager Tensors.") + raise AttributeError("op not supported for Eager Tensors.") @property def graph(self): - raise NotImplementedError("graph not supported for Eager Tensors.") + raise AttributeError("graph not supported for Eager Tensors.") @property def name(self): - raise NotImplementedError("name not supported for Eager Tensors.") - - def set_shape(self, shape): - raise NotImplementedError("set_shape not supported for Eager Tensors.") + raise AttributeError("name not supported for Eager Tensors.") @property def value_index(self): - raise NotImplementedError("value_index not supported for Eager Tensors.") + raise AttributeError("value_index not supported for Eager Tensors.") def consumers(self): raise NotImplementedError("consumers not supported for Eager Tensors.") @@ -1978,7 +2026,19 @@ class Operation(object): protocol buffer. """ # pylint: enable=line-too-long - return self._op_def + if self._c_op: + with errors.raise_exception_on_not_ok_status() as status: + with c_api_util.tf_buffer() as buf: + # pylint: disable=protected-access + c_api.TF_GraphGetOpDef(self._graph._c_graph, + compat.as_bytes(self.type), buf, status) + # pylint: enable=protected-access + data = c_api.TF_GetBuffer(buf) + op_def = op_def_pb2.OpDef() + op_def.ParseFromString(compat.as_bytes(data)) + return op_def + else: + return self._op_def @property def traceback(self): @@ -2497,7 +2557,14 @@ class Graph(object): # A map from tensor handle to its delete op. self._handle_deleters = {} # Resource container. - self._container = "" + if context.in_graph_mode(): + self._container_prefix = "" + else: + # In Eager mode, isolate resources (particularly ResourceVariables) in + # Graphs by default. This prevents unintended variable sharing. Graph mode + # gets this kind of isolation from Sessions. + self._container_prefix = "eager-execution-%d/" % (uid(),) + self._container = self._container_prefix self._registered_ops = op_def_registry.get_registered_ops() # TODO(skyewm): fold as much of the above as possible into the C @@ -2646,7 +2713,16 @@ class Graph(object): A `VersionDef`. """ # pylint: enable=line-too-long - return self._graph_def_versions + if self._c_graph: + with errors.raise_exception_on_not_ok_status() as status: + with c_api_util.tf_buffer() as buf: + c_api.TF_GraphVersions(self._c_graph, buf, status) + data = c_api.TF_GetBuffer(buf) + version_def = versions_pb2.VersionDef() + version_def.ParseFromString(compat.as_bytes(data)) + return version_def + else: + return self._graph_def_versions @property def seed(self): @@ -3809,7 +3885,7 @@ class Graph(object): """ original_container = self._container try: - self._container = container_name + self._container = self._container_prefix + container_name yield self._container finally: self._container = original_container @@ -4263,11 +4339,18 @@ def device(device_name_or_function): Returns: A context manager that specifies the default device to use for newly created ops. + + Raises: + RuntimeError: If eager execution is enabled and a function is passed in. """ if context.in_graph_mode(): return get_default_graph().device(device_name_or_function) else: # TODO(agarwal): support device functions in EAGER mode. + if callable(device_name_or_function): + raise RuntimeError( + "tf.device does not support functions when eager execution " + "is enabled.") return context.device(device_name_or_function) @@ -4534,6 +4617,91 @@ class _DefaultGraphStack(_DefaultStack): # pylint: disable=protected-access _default_graph_stack = _DefaultGraphStack() +def enable_eager_execution(config=None, device_policy=None): + """Enables, for the rest of the lifetime of this program, eager execution. + + If not called immediately on startup risks creating breakage and bugs. + + Example: + ```python + tfe.enable_eager_execution() + + # After eager execution is enabled, operations are executed as they are + # defined and `Tensor`s hold concrete values, which can be accessed as + # `numpy.ndarray`s through the `numpy()` method. + assert tf.multiply(6, 7).numpy() == 42 + ``` + + Args: + config: (Optional.) A `ConfigProto` protocol buffer with configuration + options for the Context. Note that a lot of these options may be + currently unimplemented or irrelevant when eager execution is enabled. + device_policy: (Optional.) What policy to use when trying to run an + operation on a device with inputs which are not on that device. + Valid values: + tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not + correct. + tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the + right device but raises a warning. + tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might + hide performance problems. + + Raises: + ValueError: If trying to create a context after using graph operations + or if trying to create a context with nontrivial options which differ + from those of the existing context. + """ + # pylint: disable=protected-access + if context._default_mode == context.GRAPH_MODE: + graph_mode_has_been_used = ( + _default_session_stack.stack or + _default_graph_stack._global_default_graph is not None) + if graph_mode_has_been_used: + raise ValueError( + "tfe.enable_eager_execution has to be called at program startup.") + context._default_mode = context.EAGER_MODE + if context._context is None: + context._context = context.Context(config=config, + device_policy=device_policy) + elif ((config is not None and config is not context._context._config) + or (device_policy is not None + and device_policy is not context._context._device_policy)): + raise ValueError("Trying to change the options of an active eager" + " execution. Context config: %s, specified config:" + " %s. Context device policy: %s; specified device" + " policy: %s." % (config, context._context._config, + device_policy, + context._context._device_policy)) + + +def eager_run(main=None, argv=None): + """Runs the program with an optional main function and argv list. + + The program will run with eager execution enabled. + + Example: + ```python + import tensorflow as tf + # Import subject to future changes: + from tensorflow.contrib.eager.python import tfe + + def main(_): + u = tf.constant(6.0) + v = tf.constant(7.0) + print(u * v) + + if __name__ == "__main__": + tfe.run() + ``` + + Args: + main: the main function to run. + argv: the arguments to pass to it. + """ + enable_eager_execution() + app.run(main, argv) + + def reset_default_graph(): """Clears the default graph stack and resets the global default graph. @@ -4570,6 +4738,24 @@ def get_default_graph(): return _default_graph_stack.get_default() +def get_name_scope(): + """Returns the current name scope in the default_graph. + + For example: + + ```python + with tf.name_scope('scope1'): + with tf.name_scope('scope2'): + print(tf.get_name_scope()) + ``` + would print the string `scope1/scope2`. + + Returns: + A string representing the current name scope. + """ + return get_default_graph().get_name_scope() + + def _assert_same_graph(original_item, item): """Fail if the 2 items are from different graphs. @@ -4788,9 +4974,10 @@ class GraphKeys(object): @decorator_utils.classproperty def VARIABLES(cls): # pylint: disable=no-self-argument - logging.warning("VARIABLES collection name is deprecated, " - "please use GLOBAL_VARIABLES instead; " - "VARIABLES will be removed after 2017-03-02.") + logging.log_first_n(logging.WARN, + "VARIABLES collection name is deprecated, please use " + "GLOBAL_VARIABLES instead; VARIABLES will be removed " + "after 2017-03-02.", 1) return cls.GLOBAL_VARIABLES @@ -4804,6 +4991,10 @@ def add_to_collection(name, value): name: The key for the collection. For example, the `GraphKeys` class contains many standard names for collections. value: The value to add to the collection. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ get_default_graph().add_to_collection(name, value) @@ -4818,6 +5009,10 @@ def add_to_collections(names, value): names: The key for the collections. The `GraphKeys` class contains many standard names for collections. value: The value to add to the collections. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ get_default_graph().add_to_collections(names, value) @@ -4837,6 +5032,10 @@ def get_collection_ref(key): list if no value has been added to that collection. Note that this returns the collection list itself, which can be modified in place to change the collection. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ return get_default_graph().get_collection_ref(key) @@ -4861,6 +5060,10 @@ def get_collection(key, scope=None): an empty list if no value has been added to that collection. The list contains the values in the order under which they were collected. + + @compatibility(eager) + Collections are not supported when eager execution is enabled. + @end_compatibility """ return get_default_graph().get_collection(key, scope) @@ -4895,6 +5098,10 @@ class name_scope(object): # pylint: disable=invalid-name ``` """ + @property + def name(self): + return self._name + def __init__(self, name, default_name=None, values=None): """Initialize the context manager. diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py index f20c808cdeb87037cd526faeced8f78541faa356..b1269b84bd2d3d2b2d27d559b89f49b117ddee90 100644 --- a/tensorflow/python/framework/ops_test.py +++ b/tensorflow/python/framework/ops_test.py @@ -504,6 +504,21 @@ class OperationTest(test_util.TensorFlowTestCase): r"num of inputs: 0\) does not have input 1"): x.op._update_input(1, x) # pylint: disable=protected-access + def testOpDef(self): + x = constant_op.constant(0) + y = constant_op.constant(1) + z = x + y + + # Pure Python mode doesn't create OpDefs for constants + if ops._USE_C_API: + self.assertEqual(x.op.op_def.name, "Const") + self.assertEqual(len(x.op.op_def.input_arg), 0) + self.assertEqual(len(x.op.op_def.output_arg), 1) + + self.assertEqual(z.op.op_def.name, "Add") + self.assertEqual(len(z.op.op_def.input_arg), 2) + self.assertEqual(len(z.op.op_def.output_arg), 1) + @test_util.with_c_api class CreateOpTest(test_util.TensorFlowTestCase): @@ -1627,17 +1642,16 @@ class KernelLabelTest(test_util.TensorFlowTestCase): self.assertAllEqual(b"My label is: overload_2", overload_2.eval()) +@test_util.with_c_api class AsGraphDefTest(test_util.TensorFlowTestCase): def testGraphDefVersion(self): """Test that the graphdef version is plumbed through to kernels.""" - for version in range(versions.GRAPH_DEF_VERSION_MIN_PRODUCER, - versions.GRAPH_DEF_VERSION + 2): - with ops.Graph().as_default() as g: - g.graph_def_versions.producer = version - with self.test_session(graph=g): - v = test_ops.graph_def_version().eval() - self.assertEqual(version, v) + with ops.Graph().as_default() as g: + version = g.graph_def_versions.producer + with self.test_session(graph=g): + v = test_ops.graph_def_version().eval() + self.assertEqual(version, v) def testAddShapes(self): with ops.Graph().as_default() as g: diff --git a/tensorflow/python/framework/tensor_util.py b/tensorflow/python/framework/tensor_util.py index 414c61e9306eeb15909cfd9f9fd96175fec41db4..7e74c19124ee7942ba90b8c22e9712e4d39f0480 100644 --- a/tensorflow/python/framework/tensor_util.py +++ b/tensorflow/python/framework/tensor_util.py @@ -23,6 +23,7 @@ import six from tensorflow.core.framework import tensor_pb2 from tensorflow.core.framework import tensor_shape_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.util import compat @@ -60,6 +61,8 @@ if _FAST_TENSOR_UTIL_AVAILABLE: np.int64: fast_tensor_util.AppendInt64ArrayToTensorProto, np.uint8: fast_tensor_util.AppendUInt8ArrayToTensorProto, np.uint16: fast_tensor_util.AppendUInt16ArrayToTensorProto, + np.uint32: fast_tensor_util.AppendUInt32ArrayToTensorProto, + np.uint64: fast_tensor_util.AppendUInt64ArrayToTensorProto, np.int8: fast_tensor_util.AppendInt8ArrayToTensorProto, np.int16: fast_tensor_util.AppendInt16ArrayToTensorProto, np.complex64: fast_tensor_util.AppendComplex64ArrayToTensorProto, @@ -89,11 +92,17 @@ else: def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values): tensor_proto.int_val.extend([np.asscalar(x) for x in proto_values]) + def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values]) + def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values): tensor_proto.int_val.extend([np.asscalar(x[0]) for x in proto_values]) - def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values): - tensor_proto.int64_val.extend([np.asscalar(x) for x in proto_values]) + def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.uint32_val.extend([np.asscalar(x) for x in proto_values]) + + def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.uint64_val.extend([np.asscalar(x) for x in proto_values]) def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values): tensor_proto.scomplex_val.extend([np.asscalar(v) @@ -119,6 +128,8 @@ else: np.int64: SlowAppendInt64ArrayToTensorProto, np.uint8: SlowAppendIntArrayToTensorProto, np.uint16: SlowAppendIntArrayToTensorProto, + np.uint32: SlowAppendUInt32ArrayToTensorProto, + np.uint64: SlowAppendUInt64ArrayToTensorProto, np.int8: SlowAppendIntArrayToTensorProto, np.int16: SlowAppendIntArrayToTensorProto, np.complex64: SlowAppendComplex64ArrayToTensorProto, @@ -189,7 +200,7 @@ def _FlattenToStrings(nested_strings): _TENSOR_CONTENT_TYPES = frozenset([ dtypes.float32, dtypes.float64, dtypes.int32, dtypes.uint8, dtypes.int16, dtypes.int8, dtypes.int64, dtypes.qint8, dtypes.quint8, dtypes.qint16, - dtypes.quint16, dtypes.qint32, + dtypes.quint16, dtypes.qint32, dtypes.uint32, dtypes.uint64 ]) @@ -362,10 +373,15 @@ def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): nparray = values.astype(dtype.as_numpy_dtype) else: nparray = values - elif callable(getattr(values, "__array__", None)): - # If a class has the __array__ method, then it is possible to convert - # to numpy array. + elif callable(getattr(values, "__array__", None)) or isinstance( + getattr(values, "__array_interface__", None), dict): + # If a class has the __array__ method, or __array_interface__ dict, then it + # is possible to convert to numpy array. nparray = np.asarray(values, dtype=dtype) + + # This is the preferred way to create an array from the object, so replace + # the `values` with the array so that _FlattenToStrings is not run. + values = nparray else: if values is None: raise ValueError("None values not supported.") @@ -764,6 +780,10 @@ def constant_value_as_shape(tensor): # pylint: disable=invalid-name Returns: A `TensorShape` based on the constant value of the given `tensor`. """ + if context.in_eager_mode(): + return tensor_shape.as_shape( + [dim if dim != -1 else None for dim in tensor.numpy()]) + shape = tensor.get_shape().with_rank(1) if tensor.get_shape() == [0]: return tensor_shape.scalar() diff --git a/tensorflow/python/framework/tensor_util_test.py b/tensorflow/python/framework/tensor_util_test.py index c4937de936fb9137d4e8c1ca38c2b916882f74e9..b4f28cfce0d1897c2b3be649971a8ddc06f6998d 100644 --- a/tensorflow/python/framework/tensor_util_test.py +++ b/tensorflow/python/framework/tensor_util_test.py @@ -26,6 +26,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_state_ops from tensorflow.python.ops import math_ops @@ -492,6 +493,45 @@ class TensorUtilTest(test.TestCase): self.assertEquals(np.object, a.dtype) self.assertAllEqual(np.array([[b"a", b"ab"], [b"abc", b"abcd"]]), a) + def testArrayMethod(self): + + class Wrapper(object): + + def __array__(self): + return np.array([b"foo", b"bar", b"baz"]) + + t = tensor_util.make_tensor_proto(Wrapper(), shape=[1, 3]) + self.assertProtoEquals(""" + dtype: DT_STRING + tensor_shape { dim { size: 1 } dim { size: 3 } } + string_val: "foo" + string_val: "bar" + string_val: "baz" + """, t) + a = tensor_util.MakeNdarray(t) + self.assertEquals(np.object, a.dtype) + self.assertAllEqual(np.array([[b"foo", b"bar", b"baz"]]), a) + + def testArrayInterface(self): + + class Wrapper(object): + + @property + def __array_interface__(self): + return np.array([b"foo", b"bar", b"baz"]).__array_interface__ + + t = tensor_util.make_tensor_proto(Wrapper(), shape=[1, 3]) + self.assertProtoEquals(""" + dtype: DT_STRING + tensor_shape { dim { size: 1 } dim { size: 3 } } + string_val: "foo" + string_val: "bar" + string_val: "baz" + """, t) + a = tensor_util.MakeNdarray(t) + self.assertEquals(np.object, a.dtype) + self.assertAllEqual(np.array([[b"foo", b"bar", b"baz"]]), a) + def testStringTuple(self): t = tensor_util.make_tensor_proto((b"a", b"ab", b"abc", b"abcd")) self.assertProtoEquals(""" @@ -862,6 +902,7 @@ class ConstantValueTest(test.TestCase): class ConstantValueAsShapeTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testConstant(self): np_val = np.random.rand(3).astype(np.int32) tf_val = constant_op.constant(np_val) @@ -874,11 +915,18 @@ class ConstantValueAsShapeTest(test.TestCase): tensor_shape.TensorShape([]), tensor_util.constant_value_as_shape(tf_val)) + @test_util.run_in_graph_and_eager_modes() def testShape(self): tf_val = array_ops.shape(constant_op.constant(0.0, shape=[1, 2, 3])) c_val = tensor_util.constant_value_as_shape(tf_val) self.assertEqual(tensor_shape.TensorShape([1, 2, 3]), c_val) + @test_util.run_in_graph_and_eager_modes() + def testMinusOneBecomesNone(self): + tf_val = constant_op.constant([-1, 1, -1], shape=[3]) + c_val = tensor_util.constant_value_as_shape(tf_val) + self.assertEqual([None, 1, None], c_val.as_list()) + def testPack(self): tf_val = array_ops.stack( [constant_op.constant(16), 37, array_ops.placeholder(dtypes.int32)]) diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index c681ffb514cc6bb9a9984cbc2c667644794ab1e5..e545f6de8e66e2bf062249f4221fa340965ac69c 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -47,6 +47,7 @@ from tensorflow.python import pywrap_tensorflow from tensorflow.python.client import device_lib from tensorflow.python.client import session from tensorflow.python.eager import context +from tensorflow.python.eager import tape from tensorflow.python.framework import device as pydev from tensorflow.python.framework import errors from tensorflow.python.framework import ops @@ -391,6 +392,66 @@ def with_c_api(cls): return cls +class IsolateTest(object): + """A context manager which isolates resources in its block. + + Provides an Eager-agnostic abstraction for preventing the sharing of + variables and other resources. + + In graph mode, resource handle ops are only executed in a particular Session, + isolating them from resources with the same name in other Graphs. In Eager, + separate Sessions do not exist, so resources (particularly ResourceVariables) + would be shared implicitly if a resource of the same name were created + anywhere in a Python process. Multiple handles to the same resource would + cause several issues, and so this type of sharing will raise an exception. + + Using resources with the same name in a single Python process may be useful + (especially for unit tests), so this context manager provides an abstraction + for isolating resources. Using a resource created in one Isolation environment + in another is an error. + + Example usage in Eager mode: + + ```python + import tensorflow as tf + # Import subject to change + from tensorflow.contrib.eager.python import tfe + + tfe.enable_eager_execution() + + for hyperparameter in [1, 2, 3]: + with tfe.IsolateTest(): + v = tfe.Variable(name="v", initial_value=hyperparameter) + # train model, test results ... + ``` + + IsolateTest is currently exposed through contrib.eager, but it creates a new + default Graph and provides equivalent safety in graph mode. + """ + + def __init__(self): + if context.in_eager_mode() and tape.could_possibly_record(): + raise ValueError("Cannot isolate Eager execution with an active tape.") + # In Eager, Graphs set a container which isolates resources, and maintain a + # VariableStore which caches ResourceVariable objects created through + # get_variable. So setting the default Graph has the side effect of + # isolating Eager resources. + with context.eager_mode(): + # Create the graph in Eager mode, as this provides stricter semantics + # (i.e. has a unique container prefix). This prevents implicit sharing + # when a Graph-mode graph is created and then Eager mode is enabled (an + # error through enable_eager_execution, but common with context managers + # in unit tests). + self._graph_as_default_context_manager = ops.Graph().as_default() + + def __enter__(self): + self._graph_as_default_context_manager.__enter__() + + def __exit__(self, type_arg, value_arg, traceback_arg): + return self._graph_as_default_context_manager.__exit__( + type_arg, value_arg, traceback_arg) + + def run_in_graph_and_eager_modes(__unused__=None, graph=None, config=None, use_gpu=False, force_gpu=False, reset_test=True): @@ -440,9 +501,8 @@ def run_in_graph_and_eager_modes(__unused__=None, graph=None, config=None, with context.device("/device:CPU:0"): f(self, **kwargs) - eager_graph = graph or ops.Graph() with context.eager_mode(): - with eager_graph.as_default(): + with IsolateTest(): run_eager_mode() return decorated @@ -623,8 +683,10 @@ class TensorFlowTestCase(googletest.TestCase): elif isinstance(tensors, dict): assert not tensors, "Only support empty dict now." return dict() + elif tensors is None: + return None else: - raise ValueError("Unsupported type.") + raise ValueError("Unsupported type %s." % type(tensors)) def evaluate(self, tensors): """Evaluates tensors and returns numpy values. diff --git a/tensorflow/python/framework/test_util_test.py b/tensorflow/python/framework/test_util_test.py index 6129fa2e0d06e3ac271ace515a0e3ab8fb98ac9d..b2f8d62095f75ba55344a63401525ea998a70b47 100644 --- a/tensorflow/python/framework/test_util_test.py +++ b/tensorflow/python/framework/test_util_test.py @@ -27,12 +27,16 @@ from google.protobuf import text_format from tensorflow.core.framework import graph_pb2 from tensorflow.core.protobuf import meta_graph_pb2 +from tensorflow.python.client import session +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import random_ops +from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import variables from tensorflow.python.platform import googletest @@ -325,5 +329,72 @@ class TestUtilTest(test_util.TensorFlowTestCase): self.assertEqual(a_rand, b_rand) +@test_util.with_c_api +class IsolationTest(test_util.TensorFlowTestCase): + + @test_util.run_in_graph_and_eager_modes() + def test_variable_reuse_exception(self): + with test_util.IsolateTest(), session.Session(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="first_container_variable", + initial_value=1) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + with test_util.IsolateTest(): + if context.in_graph_mode(): + with self.assertRaises(RuntimeError): + self.evaluate(first_container_variable.read_value()) + else: + with self.assertRaises(ValueError): + first_container_variable.read_value() + + @test_util.run_in_graph_and_eager_modes() + def test_variable_reuse_exception_nested(self): + with test_util.IsolateTest(), session.Session(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="first_container_variable", + initial_value=1) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + with test_util.IsolateTest(), session.Session(): + if context.in_graph_mode(): + with self.assertRaises(RuntimeError): + self.evaluate(first_container_variable.read_value()) + else: + with self.assertRaises(ValueError): + first_container_variable.read_value() + + @test_util.run_in_graph_and_eager_modes() + def test_no_sharing(self): + with test_util.IsolateTest(), session.Session(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="same_name", + initial_value=1) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + with test_util.IsolateTest(), session.Session(): + second_container_variable = resource_variable_ops.ResourceVariable( + name="same_name", + initial_value=2) + if context.in_graph_mode(): + self.evaluate([variables.global_variables_initializer()]) + self.assertEqual( + 2, self.evaluate(second_container_variable.read_value())) + self.assertEqual(1, self.evaluate(first_container_variable.read_value())) + + def test_graph_mode_isolation(self): + with context.graph_mode(): + # Even if we've (accidentally) called IsolateTest in Graph mode, it should + # provide Eager isolation. + with test_util.IsolateTest(): + with context.eager_mode(): + first_container_variable = resource_variable_ops.ResourceVariable( + name="first_container_variable", + initial_value=1) + with context.eager_mode(): + with self.assertRaises(ValueError): + first_container_variable.read_value() + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/framework/versions.py b/tensorflow/python/framework/versions.py index f4b01635dca6f23af9133f703e10c3ff744a54e9..81529e2b1e06e70fb2839c037c555ef41bcdd291 100644 --- a/tensorflow/python/framework/versions.py +++ b/tensorflow/python/framework/versions.py @@ -24,10 +24,12 @@ from tensorflow.python import pywrap_tensorflow __version__ = pywrap_tensorflow.__version__ __git_version__ = pywrap_tensorflow.__git_version__ __compiler_version__ = pywrap_tensorflow.__compiler_version__ +__cxx11_abi_flag__ = pywrap_tensorflow.__cxx11_abi_flag__ VERSION = __version__ GIT_VERSION = __git_version__ COMPILER_VERSION = __compiler_version__ +CXX11_ABI_FLAG = __cxx11_abi_flag__ GRAPH_DEF_VERSION = pywrap_tensorflow.GRAPH_DEF_VERSION GRAPH_DEF_VERSION_MIN_CONSUMER = ( @@ -39,7 +41,9 @@ __all__ = [ "__version__", "__git_version__", "__compiler_version__", + "__cxx11_abi_flag__", "COMPILER_VERSION", + "CXX11_ABI_FLAG", "GIT_VERSION", "GRAPH_DEF_VERSION", "GRAPH_DEF_VERSION_MIN_CONSUMER", diff --git a/tensorflow/python/keras/_impl/keras/engine/topology.py b/tensorflow/python/keras/_impl/keras/engine/topology.py index d9454ee8d180f1df2c9f45bd8fd8ba9f6aac260e..f9be782f85e0d22df545bd252526fcfd47a72016 100644 --- a/tensorflow/python/keras/_impl/keras/engine/topology.py +++ b/tensorflow/python/keras/_impl/keras/engine/topology.py @@ -26,6 +26,7 @@ import os import numpy as np from six.moves import zip # pylint: disable=redefined-builtin +from tensorflow.python.eager import context from tensorflow.python.framework import tensor_shape from tensorflow.python.keras._impl.keras import backend as K from tensorflow.python.keras._impl.keras.utils import conv_utils @@ -250,6 +251,8 @@ class Layer(tf_base_layers.Layer): """ # Actually call the layer (optionally building it). output = super(Layer, self).__call__(inputs, **kwargs) + if context.in_eager_mode(): + return output # Update learning phase info. output_tensors = _to_list(output) @@ -776,7 +779,7 @@ class Network(tf_base_layers.Network, Layer): if cache_key in self._output_mask_cache: return self._output_mask_cache[cache_key] else: - _, output_masks, _ = self._run_internal_graph(inputs, masks) + _, output_masks = self._run_internal_graph(inputs, masks) return output_masks def get_config(self): diff --git a/tensorflow/python/keras/_impl/keras/integration_test.py b/tensorflow/python/keras/_impl/keras/integration_test.py index d7d20e5698afa1428dfb786f1c9b82298f250045..711003684805d3f789881d13a2a0e757973c1995 100644 --- a/tensorflow/python/keras/_impl/keras/integration_test.py +++ b/tensorflow/python/keras/_impl/keras/integration_test.py @@ -192,10 +192,12 @@ class KerasIntegrationTest(test.TestCase): model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy']) + self.assertEqual(len(model.losses), 2) + self.assertEqual(len(model.updates), 2) history = model.fit(x_train, y_train, epochs=10, batch_size=16, validation_data=(x_test, y_test), verbose=2) - self.assertGreater(history.history['val_acc'][-1], 0.85) + self.assertGreater(history.history['val_acc'][-1], 0.84) def test_vector_classification_shared_model(self): # Test that functional models that feature internal updates diff --git a/tensorflow/python/keras/_impl/keras/layers/core_test.py b/tensorflow/python/keras/_impl/keras/layers/core_test.py index 5b15895c4111fb7f69d3e187065010adb5bed534..9cdebd375c89ca6cb491e4b83c0299246acb5622 100644 --- a/tensorflow/python/keras/_impl/keras/layers/core_test.py +++ b/tensorflow/python/keras/_impl/keras/layers/core_test.py @@ -20,8 +20,11 @@ from __future__ import print_function import numpy as np +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.keras._impl import keras from tensorflow.python.keras._impl.keras import testing_utils +from tensorflow.python.ops import init_ops from tensorflow.python.platform import test @@ -198,6 +201,12 @@ class CoreLayersTest(test.TestCase): self.assertEqual(layer.kernel.constraint, k_constraint) self.assertEqual(layer.bias.constraint, b_constraint) + def test_eager_dense(self): + with context.eager_mode(): + l = keras.layers.Dense(units=3, + kernel_initializer=init_ops.zeros_initializer()) + self.assertAllEqual(l(constant_op.constant([[1.0]])), [[0., 0., 0.]]) + def test_activity_regularization(self): with self.test_session(): layer = keras.layers.ActivityRegularization(l1=0.1) diff --git a/tensorflow/python/keras/_impl/keras/models.py b/tensorflow/python/keras/_impl/keras/models.py index 6e55c429e95003cc177d8d06c9eb650eb8bb1c3f..06941e4bac07a30271ac8344cc4979d9ab8ea14b 100644 --- a/tensorflow/python/keras/_impl/keras/models.py +++ b/tensorflow/python/keras/_impl/keras/models.py @@ -420,6 +420,8 @@ class Sequential(Model): # Used by Layer base class. self._dtype = None self._activity_regularizer = None + self._per_input_losses = {} + self._per_input_updates = {} # The following properties are not actually used by Keras; # they exist for compatibility with TF's variable scoping mechanism. diff --git a/tensorflow/python/keras/_impl/keras/optimizers_test.py b/tensorflow/python/keras/_impl/keras/optimizers_test.py index b63d82f6a0ff9af3cb3761ed11fd4367e542ad06..6e9e4e6c99a6ffb0684d20ca001bba98b0d799bc 100644 --- a/tensorflow/python/keras/_impl/keras/optimizers_test.py +++ b/tensorflow/python/keras/_impl/keras/optimizers_test.py @@ -93,7 +93,10 @@ class KerasOptimizersTest(test.TestCase): def test_adadelta(self): with self.test_session(): _test_optimizer(keras.optimizers.Adadelta(), target=0.6) - _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.6) + # Accuracy seems dependent on the initialization. Even adding tf.Print + # nodes in the graph seemed to affect the initialization seed, and hence + # the accuracy. + _test_optimizer(keras.optimizers.Adadelta(decay=1e-3), target=0.4) def test_adam(self): with self.test_session(): diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index d6eba3c31afc994a204fc816af1ecb676ac44f05..63844177b72cacebc717665146c9e143517f80b8 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -462,6 +462,7 @@ tf_py_test( "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", "//tensorflow/python:logging_ops", "//tensorflow/python:math_ops", @@ -621,21 +622,6 @@ tf_py_test( ], ) -tf_py_test( - name = "random_shuffle_queue_test", - size = "small", - srcs = ["random_shuffle_queue_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:data_flow_ops", - "//tensorflow/python:errors", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - ], -) - cuda_py_test( name = "resource_variable_ops_test", size = "small", @@ -917,6 +903,21 @@ cuda_py_test( ], ) +cuda_py_test( + name = "nth_element_op_test", + size = "small", + srcs = ["nth_element_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:gradients", + "//tensorflow/python:nn_grad", + "//tensorflow/python:nn_ops", + ], +) + tf_py_test( name = "unique_op_test", size = "small", @@ -1537,43 +1538,6 @@ cuda_py_test( ], ) -cuda_py_test( - name = "multinomial_op_test", - size = "small", - srcs = ["multinomial_op_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - ], -) - -cuda_py_test( - name = "multinomial_op_big_test", - size = "medium", - srcs = ["multinomial_op_big_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/core:protos_all_py", - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:control_flow_ops", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", - "//tensorflow/python:random_ops", - ], - shard_count = 3, -) - cuda_py_test( name = "numerics_test", size = "small", @@ -1658,30 +1622,6 @@ cuda_py_test( tags = ["no_windows"], ) -cuda_py_test( - name = "random_crop_test", - size = "small", - srcs = ["random_crop_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:client_testlib", - "//tensorflow/python:random_ops", - ], -) - -cuda_py_test( - name = "random_ops_test", - size = "medium", - srcs = ["random_ops_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:random_ops", - ], -) - cuda_py_test( name = "reduce_join_op_test", size = "small", @@ -2072,13 +2012,18 @@ cuda_py_test( "//tensorflow/python:data_flow_ops_gen", "//tensorflow/python:distributed_framework_test_lib", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", + "//tensorflow/python:init_ops", "//tensorflow/python:math_ops", "//tensorflow/python:nn_grad", "//tensorflow/python:training", "//tensorflow/python:tensor_array_grad", "//tensorflow/python:tensor_array_ops", "//tensorflow/python:variables", + "//tensorflow/python:variable_scope", + "//tensorflow/python/eager:backprop", + "//tensorflow/python/eager:context", ], flaky = 1, # create_local_cluster sometimes times out. ) @@ -2358,37 +2303,6 @@ cuda_py_test( shard_count = 4, ) -cuda_py_test( - name = "random_gamma_test", - size = "medium", - srcs = ["random_gamma_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - ], - shard_count = 4, -) - -cuda_py_test( - name = "random_poisson_test", - size = "medium", - srcs = ["random_poisson_test.py"], - additional_deps = [ - "//third_party/py/numpy", - "//tensorflow/python:array_ops", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:platform", - "//tensorflow/python:random_ops", - ], -) - cuda_py_test( name = "rnn_test", size = "medium", @@ -2403,6 +2317,7 @@ cuda_py_test( "//tensorflow/python:control_flow_ops", "//tensorflow/python:data_flow_grad", "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:gradients", "//tensorflow/python:init_ops", "//tensorflow/python:nn_grad", @@ -2411,6 +2326,7 @@ cuda_py_test( "//tensorflow/python:sparse_grad", "//tensorflow/python:tensor_array_grad", "//tensorflow/python:variables", + "//tensorflow/python/eager:context", ], shard_count = 10, tags = ["no_windows"], @@ -2890,6 +2806,27 @@ tf_py_test( ], ) +tf_py_test( + name = "dataset_from_generator_op_test", + size = "small", + srcs = ["dataset_from_generator_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:dtypes", + "//tensorflow/python:errors", + "//tensorflow/python:framework_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:session", + "//tensorflow/python:sparse_tensor", + "//tensorflow/python/data/ops:dataset_ops", + "//tensorflow/python/data/util:nest", + ], +) + tf_py_test( name = "filter_dataset_op_test", size = "small", @@ -2969,7 +2906,9 @@ tf_py_test( "//tensorflow/python:dataset_ops_gen", "//tensorflow/python:dtypes", "//tensorflow/python:errors", + "//tensorflow/python:io_ops", "//tensorflow/python:framework_ops", + "//tensorflow/python:parsing_ops", "//tensorflow/python:platform", "//tensorflow/python:tensor_shape", "//tensorflow/python:variables", @@ -2990,7 +2929,9 @@ tf_py_test( "//tensorflow/python:dtypes", "//tensorflow/python:errors", "//tensorflow/python:framework_ops", + "//tensorflow/python:io_ops", "//tensorflow/python:lib", + "//tensorflow/python:parsing_ops", "//tensorflow/python:tensor_shape", "//tensorflow/python:util", "//tensorflow/python/data/ops:iterator_ops", @@ -3105,6 +3046,7 @@ tf_py_test( "//tensorflow/python:function", "//tensorflow/python:functional_ops", "//tensorflow/python:gradients", + "//tensorflow/python:io_ops", "//tensorflow/python:math_ops", "//tensorflow/python:parsing_ops", "//tensorflow/python:script_ops", @@ -3131,7 +3073,10 @@ tf_py_test( "//tensorflow/python/data/ops:dataset_ops", "//tensorflow/python/data/ops:iterator_ops", ], - tags = ["no_windows"], + tags = [ + "no_oss", # Test flaky due to port collisions. + "no_windows", + ], ) filegroup( diff --git a/tensorflow/python/kernel_tests/batchtospace_op_test.py b/tensorflow/python/kernel_tests/batchtospace_op_test.py index 8ec93119f20af4c43c0aa9ae7b4a6e7540c22303..0c802476a0e788aff3de84ab736fa8f1de5daab4 100644 --- a/tensorflow/python/kernel_tests/batchtospace_op_test.py +++ b/tensorflow/python/kernel_tests/batchtospace_op_test.py @@ -24,6 +24,7 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -52,14 +53,15 @@ class BatchToSpaceDepthToSpace(test.TestCase, PythonOpImpl): def testDepthToSpaceTranspose(self): x = np.arange(20 * 5 * 8 * 7, dtype=np.float32).reshape([20, 5, 8, 7]) block_size = 2 - crops = np.zeros((2, 2), dtype=np.int32) - y1 = self.batch_to_space(x, crops, block_size=block_size) - y2 = array_ops.transpose( - array_ops.depth_to_space( - array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size), - [3, 1, 2, 0]) - with self.test_session(): - self.assertAllEqual(y1.eval(), y2.eval()) + for crops_dtype in [dtypes.int64, dtypes.int32]: + crops = array_ops.zeros((2, 2), dtype=crops_dtype) + y1 = self.batch_to_space(x, crops, block_size=block_size) + y2 = array_ops.transpose( + array_ops.depth_to_space( + array_ops.transpose(x, [3, 1, 2, 0]), block_size=block_size), + [3, 1, 2, 0]) + with self.test_session(): + self.assertAllEqual(y1.eval(), y2.eval()) class BatchToSpaceDepthToSpaceCpp(BatchToSpaceDepthToSpace, CppOpImpl): @@ -287,9 +289,10 @@ class BatchToSpaceGradientCppTest(BatchToSpaceGradientTest, CppOpImpl): class BatchToSpaceNDGradientTest(test.TestCase): # Check the gradients. - def _checkGrad(self, x, block_shape, crops): + def _checkGrad(self, x, block_shape, crops, crops_dtype): block_shape = np.array(block_shape) - crops = np.array(crops).reshape((len(block_shape), 2)) + crops = constant_op.constant( + np.array(crops).reshape((len(block_shape), 2)), crops_dtype) with self.test_session(): tf_x = ops.convert_to_tensor(x) tf_y = array_ops.batch_to_space_nd(tf_x, block_shape, crops) @@ -304,23 +307,26 @@ class BatchToSpaceNDGradientTest(test.TestCase): self.assertAllClose(x_jacob_t, x_jacob_n, rtol=1e-2, atol=epsilon) - def _compare(self, input_shape, block_shape, crops): + def _compare(self, input_shape, block_shape, crops, crops_dtype): input_shape = list(input_shape) input_shape[0] *= np.prod(block_shape) x = np.random.normal( 0, 1, np.prod(input_shape)).astype(np.float32).reshape(input_shape) - self._checkGrad(x, block_shape, crops) + self._checkGrad(x, block_shape, crops, crops_dtype) # Don't use very large numbers as dimensions here as the result is tensor # with cartesian product of the dimensions. def testSmall(self): - self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([1, 2, 3, 5], [2, 2], [[0, 0], [0, 0]], dtype) def testSmall2(self): - self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([2, 4, 3, 2], [2, 2], [[0, 0], [0, 0]], dtype) def testSmallCrop1x1(self): - self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]]) + for dtype in [dtypes.int64, dtypes.int32]: + self._compare([1, 2, 3, 5], [2, 2], [[1, 1], [1, 1]], dtype) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/confusion_matrix_test.py b/tensorflow/python/kernel_tests/confusion_matrix_test.py index 2f56540a318d3f51b8506c3effa426c1441603cd..670a625f0f1dd84c523de8acb17f9a410d184ad5 100644 --- a/tensorflow/python/kernel_tests/confusion_matrix_test.py +++ b/tensorflow/python/kernel_tests/confusion_matrix_test.py @@ -23,6 +23,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import confusion_matrix from tensorflow.python.ops import math_ops @@ -32,6 +33,7 @@ from tensorflow.python.platform import test class ConfusionMatrixTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testExample(self): """This is a test of the example provided in pydoc.""" with self.test_session(): @@ -41,8 +43,8 @@ class ConfusionMatrixTest(test.TestCase): [0, 0, 1, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 1] - ], confusion_matrix.confusion_matrix( - labels=[1, 2, 4], predictions=[2, 2, 4]).eval()) + ], self.evaluate(confusion_matrix.confusion_matrix( + labels=[1, 2, 4], predictions=[2, 2, 4]))) def _testConfMatrix(self, labels, predictions, truth, weights=None, num_classes=None): diff --git a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py index 0dcce727a3eaea2d6816d06502270517b3aa83f3..b51d483b5b6611d9596e59fd750c496bbb9c67d3 100644 --- a/tensorflow/python/kernel_tests/dataset_constructor_op_test.py +++ b/tensorflow/python/kernel_tests/dataset_constructor_op_test.py @@ -17,8 +17,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import threading - import numpy as np from tensorflow.core.protobuf import config_pb2 @@ -153,8 +151,9 @@ class DatasetConstructorTest(test.TestCase): # pylint: disable=g-long-lambda,unnecessary-lambda def testNestedStructure(self): - components = (np.array([1, 2, 3]), (np.array([4., 5.]), np.array([6., 7.])), - np.array([8, 9, 10])) + components = (np.array([1, 2, 3], dtype=np.int64), + (np.array([4., 5.]), np.array([6., 7.])), + np.array([8, 9, 10], dtype=np.int64)) dataset = dataset_ops.Dataset.from_tensors(components) self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), @@ -227,8 +226,10 @@ class DatasetConstructorTest(test.TestCase): # Define a separate set of components with matching leading # dimension for the from-slices constructor. - components_for_slices = (np.array([1, 2, 3]), (np.array( - [4., 5., 6.]), np.array([7., 8., 9.])), np.array([10, 11, 12])) + components_for_slices = (np.array([1, 2, 3], dtype=np.int64), + (np.array([4., 5., 6.]), + np.array([7., 8., 9.])), + np.array([10, 11, 12], dtype=np.int64)) dataset = dataset_ops.Dataset.from_tensor_slices(components_for_slices) self.assertEquals((dtypes.int64, (dtypes.float64, dtypes.float64), @@ -246,7 +247,7 @@ class DatasetConstructorTest(test.TestCase): self.assertEquals([3], dataset.output_shapes["b"]) def testNonSequenceNestedStructure(self): - components = np.array([1, 2, 3]) + components = np.array([1, 2, 3], dtype=np.int64) dataset = dataset_ops.Dataset.from_tensors(components) self.assertEquals(dtypes.int64, dataset.output_types) @@ -271,256 +272,6 @@ class DatasetConstructorTest(test.TestCase): self.assertEquals(dtypes.int64, get_next.dtype) self.assertEquals([3], get_next.shape) - def _testFromGenerator(self, generator, elem_sequence, num_repeats): - iterator = ( - dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) - .repeat(num_repeats) - .prefetch(5) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - for _ in range(2): # Run twice to test reinitialization. - sess.run(init_op) - for _ in range(num_repeats): - for elem in elem_sequence: - self.assertAllEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats): - iterator = ( - dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) - .repeat(num_repeats) - .prefetch(5) - .make_one_shot_iterator()) - get_next = iterator.get_next() - - with self.test_session() as sess: - for _ in range(num_repeats): - for elem in elem_sequence: - self.assertAllEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorUsingFunction(self): - def generator(): - for i in range(1, 100): - yield [i] * i - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - self._testFromGeneratorOneShot(generator, elem_sequence, 1) - self._testFromGeneratorOneShot(generator, elem_sequence, 5) - - def testFromGeneratorUsingList(self): - generator = lambda: [[i] * i for i in range(1, 100)] - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - - def testFromGeneratorUsingNdarray(self): - generator = lambda: np.arange(100, dtype=np.int64) - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - - def testFromGeneratorUsingGeneratorExpression(self): - # NOTE(mrry): Generator *expressions* are not repeatable (or in - # general reusable), because they eagerly evaluate the `for` - # expression as `iter(range(1, 100))` and discard the means of - # reconstructing `range(1, 100)`. Wrapping the generator - # expression in a `lambda` makes it repeatable. - generator = lambda: ([i] * i for i in range(1, 100)) - elem_sequence = list(generator()) - self._testFromGenerator(generator, elem_sequence, 1) - self._testFromGenerator(generator, elem_sequence, 5) - - def testFromMultipleConcurrentGenerators(self): - num_inner_repeats = 5 - num_outer_repeats = 100 - - def generator(): - for i in range(1, 10): - yield ([i] * i, [i, i ** 2, i ** 3]) - input_list = list(generator()) - - # The interleave transformation is essentially a flat map that - # draws from multiple input datasets concurrently (in a cyclic - # fashion). By placing `Datsaet.from_generator()` inside an - # interleave, we test its behavior when multiple iterators are - # active at the same time; by additionally prefetching inside the - # interleave, we create the possibility of parallel (modulo GIL) - # invocations to several iterators created by the same dataset. - def interleave_fn(_): - return (dataset_ops.Dataset.from_generator( - generator, output_types=(dtypes.int64, dtypes.int64), - output_shapes=([None], [3])) - .repeat(num_inner_repeats).prefetch(5)) - - iterator = ( - dataset_ops.Dataset.range(num_outer_repeats) - .interleave(interleave_fn, cycle_length=10, - block_length=len(input_list)) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - for _ in range(num_inner_repeats * num_outer_repeats): - for elem in input_list: - val0, val1 = sess.run(get_next) - self.assertAllEqual(elem[0], val0) - self.assertAllEqual(elem[1], val1) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorsRunningInParallel(self): - num_parallel_iterators = 3 - - # Define shared state that multiple iterator instances will access to - # demonstrate their concurrent activity. - lock = threading.Lock() - condition = threading.Condition(lock) - next_ticket = [0] # GUARDED_BY(lock) - - def generator(): - # NOTE(mrry): We yield one element before the barrier, because - # the current implementation of `Dataset.interleave()` must - # fetch one element from each incoming dataset to start the - # prefetching. - yield 0 - - # Define a barrier that `num_parallel_iterators` iterators must enter - # before any can proceed. Demonstrates that multiple iterators may be - # active at the same time. - condition.acquire() - ticket = next_ticket[0] - next_ticket[0] += 1 - if ticket == num_parallel_iterators - 1: - # The last iterator to join the barrier notifies the others. - condition.notify_all() - else: - # Wait until the last iterator enters the barrier. - while next_ticket[0] < num_parallel_iterators: - condition.wait() - condition.release() - - yield 1 - - # As in `testFromMultipleConcurrentGenerators()`, we use a combination of - # `Dataset.interleave()` and `Dataset.prefetch()` to cause multiple - # iterators to be active concurrently. - def interleave_fn(_): - return dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2) - - iterator = ( - dataset_ops.Dataset.range(num_parallel_iterators) - .interleave( - interleave_fn, cycle_length=num_parallel_iterators, block_length=1) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - for elem in [0, 1]: - for _ in range(num_parallel_iterators): - self.assertAllEqual(elem, sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorImplicitConversion(self): - def generator(): - yield [1] - yield [2] - yield [3] - - for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]: - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtype, output_shapes=[1]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - self.assertEqual(dtype, get_next.dtype) - - with self.test_session() as sess: - sess.run(init_op) - for expected in [[1], [2], [3]]: - next_val = sess.run(get_next) - self.assertEqual(dtype.as_numpy_dtype, next_val.dtype) - self.assertAllEqual(expected, next_val) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorTypeError(self): - def generator(): - yield np.array([1, 2, 3], dtype=np.int64) - yield np.array([4, 5, 6], dtype=np.int64) - yield "ERROR" - yield np.array([7, 8, 9], dtype=np.int64) - - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[3]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - self.assertAllEqual([1, 2, 3], sess.run(get_next)) - self.assertAllEqual([4, 5, 6], sess.run(get_next)) - with self.assertRaisesOpError(r"invalid literal for long\(\)"): - sess.run(get_next) - self.assertAllEqual([7, 8, 9], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorShapeError(self): - def generator(): - yield np.array([1, 2, 3], dtype=np.int64) - yield np.array([4, 5, 6], dtype=np.int64) - yield np.array([7, 8, 9, 10], dtype=np.int64) - yield np.array([11, 12, 13], dtype=np.int64) - - iterator = (dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64, output_shapes=[3]) - .make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - self.assertAllEqual([1, 2, 3], sess.run(get_next)) - self.assertAllEqual([4, 5, 6], sess.run(get_next)) - with self.assertRaisesOpError(r"element of shape \(3,\) was expected"): - sess.run(get_next) - self.assertAllEqual([11, 12, 13], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - - def testFromGeneratorHeterogeneous(self): - def generator(): - yield 1 - yield [2, 3] - - iterator = ( - dataset_ops.Dataset.from_generator( - generator, output_types=dtypes.int64).make_initializable_iterator()) - init_op = iterator.initializer - get_next = iterator.get_next() - - with self.test_session() as sess: - sess.run(init_op) - self.assertAllEqual(1, sess.run(get_next)) - self.assertAllEqual([2, 3], sess.run(get_next)) - with self.assertRaises(errors.OutOfRangeError): - sess.run(get_next) - def testSplitPipelineFailsWithPlacementError(self): with session.Session( target="", diff --git a/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py new file mode 100644 index 0000000000000000000000000000000000000000..f129d07b57b96b7869c84467aeb2276c93531ef8 --- /dev/null +++ b/tensorflow/python/kernel_tests/dataset_from_generator_op_test.py @@ -0,0 +1,307 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the experimental input pipeline ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import threading + +import numpy as np + +from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors +from tensorflow.python.platform import test + + +class DatasetConstructorTest(test.TestCase): + + def _testFromGenerator(self, generator, elem_sequence, num_repeats): + iterator = ( + dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) + .repeat(num_repeats) + .prefetch(5) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + for _ in range(2): # Run twice to test reinitialization. + sess.run(init_op) + for _ in range(num_repeats): + for elem in elem_sequence: + self.assertAllEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def _testFromGeneratorOneShot(self, generator, elem_sequence, num_repeats): + iterator = ( + dataset_ops.Dataset.from_generator(generator, output_types=dtypes.int64) + .repeat(num_repeats) + .prefetch(5) + .make_one_shot_iterator()) + get_next = iterator.get_next() + + with self.test_session() as sess: + for _ in range(num_repeats): + for elem in elem_sequence: + self.assertAllEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorUsingFunction(self): + def generator(): + for i in range(1, 100): + yield [i] * i + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + self._testFromGeneratorOneShot(generator, elem_sequence, 1) + self._testFromGeneratorOneShot(generator, elem_sequence, 5) + + def testFromGeneratorUsingList(self): + generator = lambda: [[i] * i for i in range(1, 100)] + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + + def testFromGeneratorUsingNdarray(self): + generator = lambda: np.arange(100, dtype=np.int64) + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + + def testFromGeneratorUsingGeneratorExpression(self): + # NOTE(mrry): Generator *expressions* are not repeatable (or in + # general reusable), because they eagerly evaluate the `for` + # expression as `iter(range(1, 100))` and discard the means of + # reconstructing `range(1, 100)`. Wrapping the generator + # expression in a `lambda` makes it repeatable. + generator = lambda: ([i] * i for i in range(1, 100)) + elem_sequence = list(generator()) + self._testFromGenerator(generator, elem_sequence, 1) + self._testFromGenerator(generator, elem_sequence, 5) + + def testFromMultipleConcurrentGenerators(self): + num_inner_repeats = 5 + num_outer_repeats = 100 + + def generator(): + for i in range(1, 10): + yield ([i] * i, [i, i ** 2, i ** 3]) + input_list = list(generator()) + + # The interleave transformation is essentially a flat map that + # draws from multiple input datasets concurrently (in a cyclic + # fashion). By placing `Datsaet.from_generator()` inside an + # interleave, we test its behavior when multiple iterators are + # active at the same time; by additionally prefetching inside the + # interleave, we create the possibility of parallel (modulo GIL) + # invocations to several iterators created by the same dataset. + def interleave_fn(_): + return (dataset_ops.Dataset.from_generator( + generator, output_types=(dtypes.int64, dtypes.int64), + output_shapes=([None], [3])) + .repeat(num_inner_repeats).prefetch(5)) + + iterator = ( + dataset_ops.Dataset.range(num_outer_repeats) + .interleave(interleave_fn, cycle_length=10, + block_length=len(input_list)) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for _ in range(num_inner_repeats * num_outer_repeats): + for elem in input_list: + val0, val1 = sess.run(get_next) + self.assertAllEqual(elem[0], val0) + self.assertAllEqual(elem[1], val1) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + # TODO(b/67868766): Reenable this when the source of flakiness is discovered. + def _testFromGeneratorsRunningInParallel(self): + num_parallel_iterators = 3 + + # Define shared state that multiple iterator instances will access to + # demonstrate their concurrent activity. + lock = threading.Lock() + condition = threading.Condition(lock) + next_ticket = [0] # GUARDED_BY(lock) + + def generator(): + # NOTE(mrry): We yield one element before the barrier, because + # the current implementation of `Dataset.interleave()` must + # fetch one element from each incoming dataset to start the + # prefetching. + yield 0 + + # Define a barrier that `num_parallel_iterators` iterators must enter + # before any can proceed. Demonstrates that multiple iterators may be + # active at the same time. + condition.acquire() + ticket = next_ticket[0] + next_ticket[0] += 1 + if ticket == num_parallel_iterators - 1: + # The last iterator to join the barrier notifies the others. + condition.notify_all() + else: + # Wait until the last iterator enters the barrier. + while next_ticket[0] < num_parallel_iterators: + condition.wait() + condition.release() + + yield 1 + + # As in `testFromMultipleConcurrentGenerators()`, we use a combination of + # `Dataset.interleave()` and `Dataset.prefetch()` to cause multiple + # iterators to be active concurrently. + def interleave_fn(_): + return dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[]).prefetch(2) + + iterator = ( + dataset_ops.Dataset.range(num_parallel_iterators) + .interleave( + interleave_fn, cycle_length=num_parallel_iterators, block_length=1) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for elem in [0, 1]: + for _ in range(num_parallel_iterators): + self.assertAllEqual(elem, sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorImplicitConversion(self): + def generator(): + yield [1] + yield [2] + yield [3] + + for dtype in [dtypes.int8, dtypes.int32, dtypes.int64]: + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtype, output_shapes=[1]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + self.assertEqual(dtype, get_next.dtype) + + with self.test_session() as sess: + sess.run(init_op) + for expected in [[1], [2], [3]]: + next_val = sess.run(get_next) + self.assertEqual(dtype.as_numpy_dtype, next_val.dtype) + self.assertAllEqual(expected, next_val) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorString(self): + def generator(): + yield "foo" + yield b"bar" + yield u"baz" + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.string, output_shapes=[]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + for expected in [b"foo", b"bar", b"baz"]: + next_val = sess.run(get_next) + self.assertAllEqual(expected, next_val) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorTypeError(self): + def generator(): + yield np.array([1, 2, 3], dtype=np.int64) + yield np.array([4, 5, 6], dtype=np.int64) + yield "ERROR" + yield np.array([7, 8, 9], dtype=np.int64) + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[3]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + self.assertAllEqual([1, 2, 3], sess.run(get_next)) + self.assertAllEqual([4, 5, 6], sess.run(get_next)) + # NOTE(mrry): Type name in message differs between Python 2 (`long`) and + # 3 (`int`). + with self.assertRaisesOpError(r"invalid literal for"): + sess.run(get_next) + self.assertAllEqual([7, 8, 9], sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorShapeError(self): + def generator(): + yield np.array([1, 2, 3], dtype=np.int64) + yield np.array([4, 5, 6], dtype=np.int64) + yield np.array([7, 8, 9, 10], dtype=np.int64) + yield np.array([11, 12, 13], dtype=np.int64) + + iterator = (dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64, output_shapes=[3]) + .make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + self.assertAllEqual([1, 2, 3], sess.run(get_next)) + self.assertAllEqual([4, 5, 6], sess.run(get_next)) + with self.assertRaisesOpError(r"element of shape \(3,\) was expected"): + sess.run(get_next) + self.assertAllEqual([11, 12, 13], sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + def testFromGeneratorHeterogeneous(self): + def generator(): + yield 1 + yield [2, 3] + + iterator = ( + dataset_ops.Dataset.from_generator( + generator, output_types=dtypes.int64).make_initializable_iterator()) + init_op = iterator.initializer + get_next = iterator.get_next() + + with self.test_session() as sess: + sess.run(init_op) + self.assertAllEqual(1, sess.run(get_next)) + self.assertAllEqual([2, 3], sess.run(get_next)) + with self.assertRaises(errors.OutOfRangeError): + sess.run(get_next) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/decode_csv_op_test.py b/tensorflow/python/kernel_tests/decode_csv_op_test.py index 7d9e57c8e59f082e13ab48b114a497887fa6dba3..fec52fa9cc7bcab1da67e797c2e121edac8c9345 100644 --- a/tensorflow/python/kernel_tests/decode_csv_op_test.py +++ b/tensorflow/python/kernel_tests/decode_csv_op_test.py @@ -34,7 +34,7 @@ class DecodeCSVOpTest(test.TestCase): out = sess.run(decode) for i, field in enumerate(out): - if field.dtype == np.float32: + if field.dtype == np.float32 or field.dtype == np.float64: self.assertAllClose(field, expected_out[i]) else: self.assertAllEqual(field, expected_out[i]) @@ -85,6 +85,17 @@ class DecodeCSVOpTest(test.TestCase): self._test(args, expected_out) + def testDouble(self): + args = { + "records": ["1.0", "-1.79e+308", '"1.79e+308"'], + "record_defaults": [np.array( + [], dtype=np.double)], + } + + expected_out = [[1.0, -1.79e+308, 1.79e+308]] + + self._test(args, expected_out) + def testInt64(self): args = { "records": ["1", "2", '"2147483648"'], diff --git a/tensorflow/python/kernel_tests/determinant_op_test.py b/tensorflow/python/kernel_tests/determinant_op_test.py index 7368fbc4a1a0d7e07f0ccfa572ed9247fc828c8a..222038b22ef3c766efd14fd9b1c9044a0b6e9125 100644 --- a/tensorflow/python/kernel_tests/determinant_op_test.py +++ b/tensorflow/python/kernel_tests/determinant_op_test.py @@ -126,11 +126,10 @@ class DeterminantOpTest(test.TestCase): self._compareDeterminant( np.random.rand(3, 4, 5, 2, 2).astype(np.complex128)) - def testOverflow(self): + def testInfiniteDeterminant(self): max_double = np.finfo("d").max huge_matrix = np.array([[max_double, 0.0], [0.0, max_double]]) - with self.assertRaisesOpError("not finite"): - self._compareDeterminant(huge_matrix) + self._compareDeterminant(huge_matrix) def testNonSquareMatrix(self): # When the determinant of a non-square matrix is attempted we should return diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py index f0b788573269eddfb784ad8dbb15aa8aa9932374..6cfa9b37fe0e40f4f0e5e2ad2686819e5f6d4f12 100644 --- a/tensorflow/python/kernel_tests/diag_op_test.py +++ b/tensorflow/python/kernel_tests/diag_op_test.py @@ -279,7 +279,7 @@ class MatrixDiagPartTest(test.TestCase): class DiagTest(test.TestCase): - def diagOp(self, diag, dtype, expected_ans, use_gpu=False): + def _diagOp(self, diag, dtype, expected_ans, use_gpu): with self.test_session(use_gpu=use_gpu): tf_ans = array_ops.diag(ops.convert_to_tensor(diag.astype(dtype))) out = tf_ans.eval() @@ -290,6 +290,10 @@ class DiagTest(test.TestCase): self.assertShapeEqual(expected_ans, tf_ans) self.assertShapeEqual(diag, tf_ans_inv) + def diagOp(self, diag, dtype, expected_ans): + self._diagOp(diag, dtype, expected_ans, False) + self._diagOp(diag, dtype, expected_ans, True) + def testEmptyTensor(self): x = np.array([]) expected_ans = np.empty([0, 0]) @@ -400,13 +404,53 @@ class DiagTest(test.TestCase): dtype=dtype) self.diagOp(x, dtype, expected_ans) + def testRankFourNumberTensor(self): + for dtype in [np.float32, np.float64, np.int64, np.int32]: + # Input with shape [2, 1, 2, 3] + x = np.array([[[[ 1, 2, 3], + [ 4, 5, 6]]], + [[[ 7, 8, 9], + [10, 11, 12]]]], dtype=dtype) + # Output with shape [2, 1, 2, 3, 2, 1, 2, 3] + expected_ans = np.array( + [[[[[[[[1, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 2, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 3], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]]], + [[[[[0, 0, 0], [4, 0, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 5, 0]]], + [[[0, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 6]]], + [[[0, 0, 0], [0, 0, 0]]]]]]], + + [[[[[[[0, 0, 0], [0, 0, 0]]], + [[[7, 0, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 8, 0], [0, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 9], [0, 0, 0]]]]], + [[[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [10, 0, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 11, 0]]]], + [[[[0, 0, 0], [0, 0, 0]]], + [[[0, 0, 0], [0, 0, 12]]]]]]]], dtype=dtype) + self.diagOp(x, dtype, expected_ans) + + def testInvalidRank(self): + with self.assertRaisesRegexp(ValueError, "must be at least rank 1"): + array_ops.diag(0.0) + class DiagPartOpTest(test.TestCase): def setUp(self): np.random.seed(0) - def diagPartOp(self, tensor, dtype, expected_ans, use_gpu=False): + def _diagPartOp(self, tensor, dtype, expected_ans, use_gpu): with self.test_session(use_gpu=use_gpu): tensor = ops.convert_to_tensor(tensor.astype(dtype)) tf_ans_inv = array_ops.diag_part(tensor) @@ -414,6 +458,10 @@ class DiagPartOpTest(test.TestCase): self.assertAllClose(inv_out, expected_ans) self.assertShapeEqual(expected_ans, tf_ans_inv) + def diagPartOp(self, tensor, dtype, expected_ans): + self._diagPartOp(tensor, dtype, expected_ans, False) + self._diagPartOp(tensor, dtype, expected_ans, True) + def testRankTwoFloatTensor(self): x = np.random.rand(3, 3) i = np.arange(3) @@ -451,11 +499,23 @@ class DiagPartOpTest(test.TestCase): self.diagPartOp(x, np.float32, expected_ans) self.diagPartOp(x, np.float64, expected_ans) + def testRankEightComplexTensor(self): + x = np.random.rand(2, 2, 2, 3, 2, 2, 2, 3) + i = np.arange(2)[:, None, None, None] + j = np.arange(2)[:, None, None] + k = np.arange(2)[:, None] + l = np.arange(3) + expected_ans = x[i, j, k, l, i, j, k, l] + self.diagPartOp(x, np.complex64, expected_ans) + self.diagPartOp(x, np.complex128, expected_ans) + def testOddRank(self): w = np.random.rand(2) x = np.random.rand(2, 2, 2) self.assertRaises(ValueError, self.diagPartOp, w, np.float32, 0) self.assertRaises(ValueError, self.diagPartOp, x, np.float32, 0) + with self.assertRaises(ValueError): + array_ops.diag_part(0.0) def testUnevenDimensions(self): w = np.random.rand(2, 5) diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py index 21fe588ac144853551c3e3d0a7b93539f2596151..f5717a5a21a0be82382c5da556ed6f5540591abf 100644 --- a/tensorflow/python/kernel_tests/functional_ops_test.py +++ b/tensorflow/python/kernel_tests/functional_ops_test.py @@ -52,6 +52,7 @@ def simple_scoped_fn(a, x): class FunctionalOpsTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testFoldl_Simple(self): with self.test_session(): elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data") @@ -59,13 +60,13 @@ class FunctionalOpsTest(test.TestCase): r = functional_ops.foldl( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems) - self.assertAllEqual(208, r.eval()) + self.assertAllEqual(208, self.evaluate(r)) r = functional_ops.foldl( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems, initializer=10) - self.assertAllEqual(880, r.eval()) + self.assertAllEqual(880, self.evaluate(r)) def testFoldl_Scoped(self): with self.test_session() as sess: @@ -78,14 +79,15 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(variables.trainable_variables()[0].name, "root/body/two:0") sess.run([variables.global_variables_initializer()]) - self.assertAllEqual(208, r.eval()) + self.assertAllEqual(208, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.foldl(simple_scoped_fn, elems, initializer=10) self.assertEqual(len(variables.trainable_variables()), 1) - self.assertAllEqual(880, r.eval()) + self.assertAllEqual(880, self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testFoldr_Simple(self): with self.test_session(): elems = constant_op.constant([1, 2, 3, 4, 5, 6], name="data") @@ -93,13 +95,13 @@ class FunctionalOpsTest(test.TestCase): r = functional_ops.foldr( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems) - self.assertAllEqual(450, r.eval()) + self.assertAllEqual(450, self.evaluate(r)) r = functional_ops.foldr( lambda a, x: math_ops.multiply(math_ops.add(a, x), 2), elems, initializer=10) - self.assertAllEqual(1282, r.eval()) + self.assertAllEqual(1282, self.evaluate(r)) def testFoldr_Scoped(self): with self.test_session() as sess: @@ -112,13 +114,13 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(variables.trainable_variables()[0].name, "root/body/two:0") sess.run([variables.global_variables_initializer()]) - self.assertAllEqual(450, r.eval()) + self.assertAllEqual(450, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.foldr(simple_scoped_fn, elems, initializer=10) self.assertEqual(len(variables.trainable_variables()), 1) - self.assertAllEqual(1282, r.eval()) + self.assertAllEqual(1282, self.evaluate(r)) # pylint: disable=unnecessary-lambda def testFold_Grad(self): @@ -128,21 +130,23 @@ class FunctionalOpsTest(test.TestCase): r = functional_ops.foldl( lambda a, x: math_ops.multiply(a, x), elems, initializer=v) r = gradients_impl.gradients(r, v)[0] - self.assertAllEqual(720.0, r.eval()) + self.assertAllEqual(720.0, self.evaluate(r)) r = functional_ops.foldr( lambda a, x: math_ops.multiply(a, x), elems, initializer=v) r = gradients_impl.gradients(r, v)[0] - self.assertAllEqual(720.0, r.eval()) + self.assertAllEqual(720.0, self.evaluate(r)) # pylint: enable=unnecessary-lambda + @test_util.run_in_graph_and_eager_modes() def testMap_Simple(self): with self.test_session(): nums = [1, 2, 3, 4, 5, 6] elems = constant_op.constant(nums, name="data") r = functional_ops.map_fn( lambda x: math_ops.multiply(math_ops.add(x, 3), 2), elems) - self.assertAllEqual(np.array([(x + 3) * 2 for x in nums]), r.eval()) + self.assertAllEqual( + np.array([(x + 3) * 2 for x in nums]), self.evaluate(r)) def testMapSparseTensor(self): with self.test_session(): @@ -177,13 +181,13 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(variables.trainable_variables()[0].name, "root/body/two:0") sess.run([variables.global_variables_initializer()]) - self.assertAllEqual(doubles, r.eval()) + self.assertAllEqual(doubles, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.map_fn(double_scoped, elems) self.assertEqual(len(variables.trainable_variables()), 1) - self.assertAllEqual(doubles, r.eval()) + self.assertAllEqual(doubles, self.evaluate(r)) def testMap_Grad(self): with self.test_session(): @@ -192,19 +196,22 @@ class FunctionalOpsTest(test.TestCase): y = functional_ops.map_fn( lambda x: math_ops.multiply(math_ops.square(x), param), elems) r = gradients_impl.gradients(y, param)[0] - self.assertAllEqual(91.0, r.eval()) + self.assertAllEqual(91.0, self.evaluate(r)) r = gradients_impl.gradients(y, elems)[0] - self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], r.eval()) + self.assertAllEqual([4.0, 8.0, 12.0, 16.0, 20.0, 24.0], self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testMap_SimpleNotTensor(self): with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) r = functional_ops.map_fn( lambda x: math_ops.multiply(math_ops.add(x, 3), 2), nums) - self.assertAllEqual(np.array([(x + 3) * 2 for x in nums]), r.eval()) + self.assertAllEqual( + np.array([(x + 3) * 2 for x in nums]), self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testMap_SingleInputMultiOutput(self): - with self.test_session() as sess: + with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) r = functional_ops.map_fn( lambda x: ((x + 3) * 2, -(x + 3) * 2), @@ -213,10 +220,11 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual(2, len(r)) self.assertEqual((6,), r[0].get_shape()) self.assertEqual((6,), r[1].get_shape()) - received = sess.run(r) + received = self.evaluate(r) self.assertAllEqual((nums + 3) * 2, received[0]) self.assertAllEqual(-(nums + 3) * 2, received[1]) + @test_util.run_in_graph_and_eager_modes() def testMap_MultiOutputMismatchedDtype(self): with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) @@ -228,6 +236,7 @@ class FunctionalOpsTest(test.TestCase): nums, dtype=[dtypes.int64, dtypes.int64]) + @test_util.run_in_graph_and_eager_modes() def testMap_MultiInputSingleOutput(self): with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) @@ -235,11 +244,12 @@ class FunctionalOpsTest(test.TestCase): lambda x: x[0] * x[1][0] + x[1][1], (nums, (nums, -nums)), dtype=dtypes.int64) self.assertEqual((6,), r.get_shape()) - received = r.eval() + received = self.evaluate(r) self.assertAllEqual(nums * nums + (-nums), received) + @test_util.run_in_graph_and_eager_modes() def testMap_MultiInputSameStructureOutput(self): - with self.test_session() as sess: + with self.test_session(): nums = np.array([1, 2, 3, 4, 5, 6]) r = functional_ops.map_fn(lambda x: (x[1][0], (x[1][1], x[0])), (nums, (2 * nums, -nums))) @@ -247,11 +257,12 @@ class FunctionalOpsTest(test.TestCase): self.assertEqual((6,), r[0].get_shape()) self.assertEqual((6,), r[1].get_shape()) self.assertEqual((6,), r[2].get_shape()) - received = sess.run(r) + received = self.evaluate(r) self.assertAllEqual(2 * nums, received[0]) self.assertAllEqual(-nums, received[1]) self.assertAllEqual(nums, received[2]) + @test_util.run_in_graph_and_eager_modes() def testScan_Simple(self): with self.test_session(): elems = constant_op.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], name="data") @@ -259,24 +270,26 @@ class FunctionalOpsTest(test.TestCase): # pylint: disable=unnecessary-lambda r = functional_ops.scan(lambda a, x: math_ops.multiply(a, x), elems) - self.assertAllEqual([1., 2., 6., 24., 120., 720.], r.eval()) + self.assertAllEqual([1., 2., 6., 24., 120., 720.], self.evaluate(r)) r = functional_ops.scan( lambda a, x: math_ops.multiply(a, x), elems, initializer=v) - self.assertAllEqual([2., 4., 12., 48., 240., 1440.], r.eval()) + self.assertAllEqual([2., 4., 12., 48., 240., 1440.], self.evaluate(r)) # pylint: enable=unnecessary-lambda + @test_util.run_in_graph_and_eager_modes() def testScan_SingleInputMultiOutput(self): - with self.test_session() as sess: + with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) initializer = (np.array(1.0), np.array(-1.0)) r = functional_ops.scan(lambda a, x: (a[0] * x, -a[1] * x), elems, initializer) - r_value = sess.run(r) + r_value = self.evaluate(r) self.assertAllEqual([1.0, 2.0, 6.0, 24.0, 120.0, 720.0], r_value[0]) self.assertAllEqual([1.0, -2.0, 6.0, -24.0, 120.0, -720.0], r_value[1]) + @test_util.run_in_graph_and_eager_modes() def testScan_MultiInputSingleOutput(self): with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) @@ -284,17 +297,19 @@ class FunctionalOpsTest(test.TestCase): # Multiply a * 1 each time r = functional_ops.scan(lambda a, x: a * (x[0] + x[1]), (elems + 1, -elems), initializer) - self.assertAllEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], r.eval()) + self.assertAllEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testScan_MultiInputSameTypeOutput(self): - with self.test_session() as sess: + with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) r = functional_ops.scan(lambda a, x: (a[0] + x[0], a[1] + x[1]), (elems, -elems)) - r_value = sess.run(r) + r_value = self.evaluate(r) self.assertAllEqual(np.cumsum(elems), r_value[0]) self.assertAllEqual(np.cumsum(-elems), r_value[1]) + @test_util.run_in_graph_and_eager_modes() def testScan_MultiOutputMismatchedInitializer(self): with self.test_session(): elems = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]) @@ -316,15 +331,16 @@ class FunctionalOpsTest(test.TestCase): "root/body/two:0") sess.run([variables.global_variables_initializer()]) results = np.array([1, 6, 18, 44, 98, 208]) - self.assertAllEqual(results, r.eval()) + self.assertAllEqual(results, self.evaluate(r)) # Now let's reuse our single variable. varscope.reuse_variables() r = functional_ops.scan(simple_scoped_fn, elems, initializer=2) self.assertEqual(len(variables.trainable_variables()), 1) results = np.array([6, 16, 38, 84, 178, 368]) - self.assertAllEqual(results, r.eval()) + self.assertAllEqual(results, self.evaluate(r)) + @test_util.run_in_graph_and_eager_modes() def testScanFoldl_Nested(self): with self.test_session(): elems = constant_op.constant([1.0, 2.0, 3.0, 4.0], name="data") @@ -346,7 +362,7 @@ class FunctionalOpsTest(test.TestCase): # t == 3, a == 2.25, x == 4 (returns 9) # t_0 == 0, b == a == 2.25, y == 0.5, returns b * y * x = 4.5 # t_1 == 1, b == 4.5, y == 0.5, returns b * y * x = 9 - self.assertAllClose([1., 1., 2.25, 9.], r.eval()) + self.assertAllClose([1., 1., 2.25, 9.], self.evaluate(r)) def testScan_Control(self): with self.test_session() as sess: @@ -369,7 +385,7 @@ class FunctionalOpsTest(test.TestCase): lambda a, x: math_ops.multiply(a, x), elems, initializer=v) # pylint: enable=unnecessary-lambda r = gradients_impl.gradients(r, v)[0] - self.assertAllEqual(873.0, r.eval()) + self.assertAllEqual(873.0, self.evaluate(r)) def testScanGradientWithPartStopGradient(self): a = variables.Variable(0.0, name="a") @@ -383,6 +399,7 @@ class FunctionalOpsTest(test.TestCase): variables.global_variables_initializer().run() sess.run(grad) + @test_util.run_in_graph_and_eager_modes() def testFoldShape(self): with self.test_session(): x = constant_op.constant([[1, 2, 3], [4, 5, 6]]) @@ -392,32 +409,37 @@ class FunctionalOpsTest(test.TestCase): initializer = constant_op.constant([0, 0, 0]) y = functional_ops.foldl(fn, x, initializer=initializer) - self.assertAllEqual(y.get_shape(), y.eval().shape) + self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) + @test_util.run_in_graph_and_eager_modes() def testMapShape(self): with self.test_session(): x = constant_op.constant([[1, 2, 3], [4, 5, 6]]) y = functional_ops.map_fn(lambda e: e, x) - self.assertAllEqual(y.get_shape(), y.eval().shape) + self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) def testMapUnknownShape(self): x = array_ops.placeholder(dtypes.float32) y = functional_ops.map_fn(lambda e: e, x) self.assertIs(None, y.get_shape().dims) + @test_util.run_in_graph_and_eager_modes() def testMapEmptyScalar(self): with self.test_session(): map_return = functional_ops.map_fn(lambda x: 1, constant_op.constant([])) self.assertAllEqual([0], map_return.get_shape().dims) - self.assertAllEqual([0], map_return.eval().shape) + self.assertAllEqual([0], self.evaluate(map_return).shape) + # TODO(akshayka): this test fails in eager: the iterable is of length 0 so + # so the body of the while loop never executes def testMapEmptyTensor(self): with self.test_session(): map_return = functional_ops.map_fn(lambda x: array_ops.zeros([3, 2]), constant_op.constant([])) self.assertAllEqual([0, 3, 2], map_return.get_shape().dims) - self.assertAllEqual([0, 3, 2], map_return.eval().shape) + self.assertAllEqual([0, 3, 2], self.evaluate(map_return).shape) + @test_util.run_in_graph_and_eager_modes() def testScanShape(self): with self.test_session(): x = constant_op.constant([[1, 2, 3], [4, 5, 6]]) @@ -427,14 +449,16 @@ class FunctionalOpsTest(test.TestCase): initializer = constant_op.constant([0, 0, 0]) y = functional_ops.scan(fn, x, initializer=initializer) - self.assertAllEqual(y.get_shape(), y.eval().shape) + self.assertAllEqual(y.get_shape(), self.evaluate(y).shape) + # TODO(akshayka): this test fails in eager: the iterable is of length 0 so + # so the body of the while loop never executes def testScanEmptyTensor(self): with self.test_session(): x = functional_ops.scan( lambda x, _: x, math_ops.range(0), initializer=array_ops.ones([2, 4])) self.assertAllEqual([0, 2, 4], x.get_shape()) - self.assertAllEqual(x.get_shape(), x.eval().shape) + self.assertAllEqual(x.get_shape(), self.evaluate(x).shape) def testScanUnknownShape(self): x = array_ops.placeholder(dtypes.float32) diff --git a/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py b/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py index d7315a252633542f5a34f3d8a2ac7708c08ecec6..45dfa13720b09c7bba979b72a339c13dcd2d827b 100644 --- a/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py +++ b/tensorflow/python/kernel_tests/iterator_ops_cluster_test.py @@ -53,13 +53,8 @@ class IteratorClusterTest(test.TestCase): with self.assertRaises(errors.InvalidArgumentError): sess.run(get_next_op) - def testRemoteIteratorUsingRemoteCallOp(self): - worker_config = config_pb2.ConfigProto() - worker_config.device_count["CPU"] = 2 - worker, _ = test_util.create_local_cluster( - 1, 1, worker_config=worker_config) - - with ops.device("/job:worker/replica:0/task:0/cpu:1"): + def _testRemoteIteratorHelper(self, device0, device1, target): + with ops.device(device1): dataset_3 = dataset_ops.Dataset.from_tensor_slices([1, 2, 3]) iterator_3 = dataset_3.make_one_shot_iterator() iterator_3_handle = iterator_3.string_handle() @@ -70,7 +65,7 @@ class IteratorClusterTest(test.TestCase): h, dataset_3.output_types, dataset_3.output_shapes) return remote_iterator.get_next() - with ops.device("/job:worker/replica:0/task:0/cpu:0"): + with ops.device(device0): target_placeholder = array_ops.placeholder(dtypes.string, shape=[]) remote_op = functional_ops.remote_call( args=[iterator_3_handle], @@ -78,32 +73,35 @@ class IteratorClusterTest(test.TestCase): f=_remote_fn, target=target_placeholder) - with session.Session(worker[0].target) as sess: - elem = sess.run( - remote_op, - feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"}) + with session.Session(target) as sess: + elem = sess.run(remote_op, feed_dict={target_placeholder: device1}) self.assertEqual(elem, [1]) # Fails when target is cpu:0 where the resource is not located. with self.assertRaises(errors.InvalidArgumentError): - sess.run( - remote_op, - feed_dict={ - target_placeholder: "/job:worker/replica:0/task:0/cpu:0" - }) - elem = sess.run( - remote_op, - feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"}) + sess.run(remote_op, feed_dict={target_placeholder: device0}) + elem = sess.run(iterator_3.get_next()) self.assertEqual(elem, [2]) - elem = sess.run( - remote_op, - feed_dict={target_placeholder: "/job:worker/replica:0/task:0/cpu:1"}) + elem = sess.run(remote_op, feed_dict={target_placeholder: device1}) self.assertEqual(elem, [3]) with self.assertRaises(errors.OutOfRangeError): - sess.run( - remote_op, - feed_dict={ - target_placeholder: "/job:worker/replica:0/task:0/cpu:1" - }) + sess.run(remote_op, feed_dict={target_placeholder: device1}) + + def testRemoteIteratorUsingRemoteCallOp(self): + worker_config = config_pb2.ConfigProto() + worker_config.device_count["CPU"] = 2 + worker, _ = test_util.create_local_cluster( + 1, 1, worker_config=worker_config) + + self._testRemoteIteratorHelper("/job:worker/replica:0/task:0/cpu:0", + "/job:worker/replica:0/task:0/cpu:1", + worker[0].target) + + def testRemoteIteratorUsingRemoteCallOpCrossProcess(self): + workers, _ = test_util.create_local_cluster(2, 1) + + self._testRemoteIteratorHelper("/job:worker/replica:0/task:0/cpu:0", + "/job:worker/replica:0/task:1/cpu:0", + workers[0].target) if __name__ == "__main__": diff --git a/tensorflow/python/kernel_tests/iterator_ops_test.py b/tensorflow/python/kernel_tests/iterator_ops_test.py index b5ec9f7db03c0ffe16bd62a696f4af72e3648bcb..2128ef4ae17668309af96c4fb21837cb7659a122 100644 --- a/tensorflow/python/kernel_tests/iterator_ops_test.py +++ b/tensorflow/python/kernel_tests/iterator_ops_test.py @@ -35,6 +35,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import functional_ops from tensorflow.python.ops import gen_dataset_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import io_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import script_ops @@ -538,9 +539,23 @@ class IteratorTest(test.TestCase): def testIncorrectIteratorRestore(self): - def _iterator_checkpoint_prefix(): + def _path(): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + _path(), parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_range_dataset_graph(): start = 1 stop = 10 @@ -548,22 +563,18 @@ class IteratorTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = _iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op def _build_reader_dataset_graph(): filenames = ["test"] # Does not exist but we don't care in this test. - path = _iterator_checkpoint_prefix() iterator = readers.FixedLengthRecordDataset( filenames, 1, 0, 0).make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = _save_op(iterator._iterator_resource) + restore_op = _restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op # Saving iterator for RangeDataset graph. diff --git a/tensorflow/python/kernel_tests/listdiff_op_test.py b/tensorflow/python/kernel_tests/listdiff_op_test.py index 4f053d2a2199b96dee322ff58942cd257700f76d..ee86cf0b249d0ff46fddd3ae09e4dee0d8603350 100644 --- a/tensorflow/python/kernel_tests/listdiff_op_test.py +++ b/tensorflow/python/kernel_tests/listdiff_op_test.py @@ -41,15 +41,17 @@ class ListDiffTest(test.TestCase): y = [compat.as_bytes(str(a)) for a in y] out = [compat.as_bytes(str(a)) for a in out] for diff_func in [array_ops.setdiff1d]: - with self.test_session() as sess: - x_tensor = ops.convert_to_tensor(x, dtype=dtype) - y_tensor = ops.convert_to_tensor(y, dtype=dtype) - out_tensor, idx_tensor = diff_func(x_tensor, y_tensor) - tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) - self.assertAllEqual(tf_out, out) - self.assertAllEqual(tf_idx, idx) - self.assertEqual(1, out_tensor.get_shape().ndims) - self.assertEqual(1, idx_tensor.get_shape().ndims) + for index_dtype in [dtypes.int32, dtypes.int64]: + with self.test_session() as sess: + x_tensor = ops.convert_to_tensor(x, dtype=dtype) + y_tensor = ops.convert_to_tensor(y, dtype=dtype) + out_tensor, idx_tensor = diff_func(x_tensor, y_tensor, + index_dtype=index_dtype) + tf_out, tf_idx = sess.run([out_tensor, idx_tensor]) + self.assertAllEqual(tf_out, out) + self.assertAllEqual(tf_idx, idx) + self.assertEqual(1, out_tensor.get_shape().ndims) + self.assertEqual(1, idx_tensor.get_shape().ndims) def testBasic1(self): x = [1, 2, 3, 4] diff --git a/tensorflow/python/kernel_tests/logging_ops_test.py b/tensorflow/python/kernel_tests/logging_ops_test.py index 7fe65c57ccc451cd56059518eba7b2ae214f8e74..28c85fa13ad100c38382d2b787ff965f9e3ca44e 100644 --- a/tensorflow/python/kernel_tests/logging_ops_test.py +++ b/tensorflow/python/kernel_tests/logging_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl from tensorflow.python.ops import logging_ops @@ -58,6 +59,7 @@ class LoggingOpsTest(test.TestCase): class PrintGradientTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testPrintShape(self): inp = constant_op.constant(2.0, shape=[100, 32]) inp_printed = logging_ops.Print(inp, [inp]) diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py index 804346e6e7b415279fec98dedd9fbcebb81d538d..e5b7cbce7aa31bb3aa288ab529ef26b9c4a0003e 100644 --- a/tensorflow/python/kernel_tests/metrics_test.py +++ b/tensorflow/python/kernel_tests/metrics_test.py @@ -1827,6 +1827,38 @@ def _test_sparse_precision_at_k(predictions, test_case.assertEqual(expected, metric.eval()) +def _test_precision_at_top_k( + predictions_idx, + labels, + expected, + k=None, + class_id=None, + weights=None, + test_case=None): + with ops.Graph().as_default() as g, test_case.test_session(g): + if weights is not None: + weights = constant_op.constant(weights, dtypes_lib.float32) + metric, update = metrics.precision_at_top_k( + predictions_idx=constant_op.constant(predictions_idx, dtypes_lib.int32), + labels=labels, + k=k, + class_id=class_id, + weights=weights) + + # Fails without initialized vars. + test_case.assertRaises(errors_impl.OpError, metric.eval) + test_case.assertRaises(errors_impl.OpError, update.eval) + variables.variables_initializer(variables.local_variables()).run() + + # Run per-step op and assert expected values. + if math.isnan(expected): + test_case.assertTrue(math.isnan(update.eval())) + test_case.assertTrue(math.isnan(metric.eval())) + else: + test_case.assertEqual(expected, update.eval()) + test_case.assertEqual(expected, metric.eval()) + + def _test_sparse_average_precision_at_k(predictions, labels, k, @@ -1858,6 +1890,7 @@ class SingleLabelSparsePrecisionTest(test.TestCase): def setUp(self): self._predictions = ((0.1, 0.3, 0.2, 0.4), (0.1, 0.2, 0.3, 0.4)) + self._predictions_idx = [[3], [3]] indicator_labels = ((0, 0, 0, 1), (0, 0, 1, 0)) class_labels = (3, 2) # Sparse vs dense, and 1d vs 2d labels should all be handled the same. @@ -1868,6 +1901,8 @@ class SingleLabelSparsePrecisionTest(test.TestCase): [[class_id] for class_id in class_labels], dtype=np.int64)) self._test_sparse_precision_at_k = functools.partial( _test_sparse_precision_at_k, test_case=self) + self._test_precision_at_top_k = functools.partial( + _test_precision_at_top_k, test_case=self) self._test_sparse_average_precision_at_k = functools.partial( _test_sparse_average_precision_at_k, test_case=self) @@ -1877,16 +1912,24 @@ class SingleLabelSparsePrecisionTest(test.TestCase): for class_id in (-1, 0, 1, 2, 4): self._test_sparse_precision_at_k( self._predictions, labels, k=1, expected=NAN, class_id=class_id) + self._test_precision_at_top_k( + self._predictions_idx, labels, k=1, expected=NAN, class_id=class_id) def test_at_k1(self): for labels in self._labels: # Class 3: 1 label, 2 predictions, 1 correct. self._test_sparse_precision_at_k( self._predictions, labels, k=1, expected=1.0 / 2, class_id=3) + self._test_precision_at_top_k( + self._predictions_idx, labels, k=1, expected=1.0 / 2, class_id=3) # All classes: 2 labels, 2 predictions, 1 correct. self._test_sparse_precision_at_k( self._predictions, labels, k=1, expected=1.0 / 2) + self._test_precision_at_top_k( + self._predictions_idx, labels, k=1, expected=1.0 / 2) + self._test_sparse_average_precision_at_k( + self._predictions, labels, k=1, expected=1.0 / 2) class MultiLabelSparsePrecisionTest(test.TestCase): @@ -1894,6 +1937,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase): def setUp(self): self._test_sparse_precision_at_k = functools.partial( _test_sparse_precision_at_k, test_case=self) + self._test_precision_at_top_k = functools.partial( + _test_precision_at_top_k, test_case=self) self._test_sparse_average_precision_at_k = functools.partial( _test_sparse_average_precision_at_k, test_case=self) @@ -1905,6 +1950,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase): labels = np.array([labels_ex1], dtype=np.int64) predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3) predictions = (predictions_ex1,) + predictions_idx_ex1 = (5, 3, 6, 0, 1) precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4) avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3, (precision_ex1[1] + precision_ex1[3]) / 4) @@ -1912,6 +1958,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase): k = i + 1 self._test_sparse_precision_at_k( predictions, labels, k, expected=precision_ex1[i]) + self._test_precision_at_top_k( + (predictions_idx_ex1[:k],), labels, k=k, expected=precision_ex1[i]) self._test_sparse_average_precision_at_k( predictions, labels, k, expected=avg_precision_ex1[i]) @@ -1920,6 +1968,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase): labels = np.array([labels_ex2], dtype=np.int64) predictions_ex2 = (0.3, 0.5, 0.0, 0.4, 0.0, 0.1, 0.2) predictions = (predictions_ex2,) + predictions_idx_ex2 = (1, 3, 0, 6, 5) precision_ex2 = (0.0 / 1, 0.0 / 2, 1.0 / 3, 2.0 / 4) avg_precision_ex2 = (0.0 / 1, 0.0 / 2, precision_ex2[2] / 3, (precision_ex2[2] + precision_ex2[3]) / 4) @@ -1927,6 +1976,8 @@ class MultiLabelSparsePrecisionTest(test.TestCase): k = i + 1 self._test_sparse_precision_at_k( predictions, labels, k, expected=precision_ex2[i]) + self._test_precision_at_top_k( + (predictions_idx_ex2[:k],), labels, k=k, expected=precision_ex2[i]) self._test_sparse_average_precision_at_k( predictions, labels, k, expected=avg_precision_ex2[i]) @@ -1942,8 +1993,11 @@ class MultiLabelSparsePrecisionTest(test.TestCase): ] for i in xrange(4): k = i + 1 + predictions_idx = (predictions_idx_ex1[:k], predictions_idx_ex2[:k]) self._test_sparse_precision_at_k( predictions, labels, k, expected=streaming_precision[i]) + self._test_precision_at_top_k( + predictions_idx, labels, k=k, expected=streaming_precision[i]) self._test_sparse_average_precision_at_k( predictions, labels, k, expected=streaming_average_precision[i]) @@ -1969,6 +2023,7 @@ class MultiLabelSparsePrecisionTest(test.TestCase): labels = np.array([labels_ex1], dtype=np.int64) predictions_ex1 = (0.2, 0.1, 0.0, 0.4, 0.0, 0.5, 0.3) predictions = (predictions_ex1,) + predictions_idx_ex1 = (5, 3, 6, 0, 1) precision_ex1 = (0.0 / 1, 1.0 / 2, 1.0 / 3, 2.0 / 4) avg_precision_ex1 = (0.0 / 1, precision_ex1[1] / 2, precision_ex1[1] / 3, (precision_ex1[1] + precision_ex1[3]) / 4) @@ -1976,12 +2031,15 @@ class MultiLabelSparsePrecisionTest(test.TestCase): k = i + 1 self._test_sparse_precision_at_k( predictions, labels, k, expected=precision_ex1[i]) + self._test_precision_at_top_k( + (predictions_idx_ex1[:k],), labels, k=k, expected=precision_ex1[i]) self._test_sparse_average_precision_at_k( predictions, labels, k, expected=avg_precision_ex1[i]) def test_three_labels_at_k5_no_predictions(self): predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] + predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]] sparse_labels = _binary_2d_label_to_2d_sparse_value( [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) @@ -1991,10 +2049,13 @@ class MultiLabelSparsePrecisionTest(test.TestCase): for class_id in (-1, 1, 3, 8, 10): self._test_sparse_precision_at_k( predictions, labels, k=5, expected=NAN, class_id=class_id) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=NAN, class_id=class_id) def test_three_labels_at_k5_no_labels(self): predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] + predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]] sparse_labels = _binary_2d_label_to_2d_sparse_value( [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) @@ -2004,10 +2065,13 @@ class MultiLabelSparsePrecisionTest(test.TestCase): for class_id in (0, 4, 6, 9): self._test_sparse_precision_at_k( predictions, labels, k=5, expected=0.0, class_id=class_id) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=0.0, class_id=class_id) def test_three_labels_at_k5(self): predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] + predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]] sparse_labels = _binary_2d_label_to_2d_sparse_value( [[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]]) dense_labels = np.array([[2, 7, 8], [1, 2, 5]], dtype=np.int64) @@ -2016,23 +2080,32 @@ class MultiLabelSparsePrecisionTest(test.TestCase): # Class 2: 2 labels, 2 correct predictions. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=2.0 / 2, class_id=2) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=2.0 / 2, class_id=2) # Class 5: 1 label, 1 correct prediction. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=1.0 / 1, class_id=5) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=1.0 / 1, class_id=5) # Class 7: 1 label, 1 incorrect prediction. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=0.0 / 1, class_id=7) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=0.0 / 1, class_id=7) # All classes: 10 predictions, 3 correct. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=3.0 / 10) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=3.0 / 10) def test_three_labels_at_k5_some_out_of_range(self): """Tests that labels outside the [0, n_classes) range are ignored.""" predictions = [[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]] + predictions_idx = [[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]] sp_labels = sparse_tensor.SparseTensorValue( indices=[[0, 0], [0, 1], [0, 2], [0, 3], [1, 0], [1, 1], [1, 2], [1, 3]], @@ -2043,24 +2116,34 @@ class MultiLabelSparsePrecisionTest(test.TestCase): # Class 2: 2 labels, 2 correct predictions. self._test_sparse_precision_at_k( predictions, sp_labels, k=5, expected=2.0 / 2, class_id=2) + self._test_precision_at_top_k( + predictions_idx, sp_labels, k=5, expected=2.0 / 2, class_id=2) # Class 5: 1 label, 1 correct prediction. self._test_sparse_precision_at_k( predictions, sp_labels, k=5, expected=1.0 / 1, class_id=5) + self._test_precision_at_top_k( + predictions_idx, sp_labels, k=5, expected=1.0 / 1, class_id=5) # Class 7: 1 label, 1 incorrect prediction. self._test_sparse_precision_at_k( predictions, sp_labels, k=5, expected=0.0 / 1, class_id=7) + self._test_precision_at_top_k( + predictions_idx, sp_labels, k=5, expected=0.0 / 1, class_id=7) # All classes: 10 predictions, 3 correct. self._test_sparse_precision_at_k( predictions, sp_labels, k=5, expected=3.0 / 10) + self._test_precision_at_top_k( + predictions_idx, sp_labels, k=5, expected=3.0 / 10) def test_3d_nan(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] + predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]], + [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]] labels = _binary_3d_label_to_sparse_value( [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) @@ -2069,12 +2152,16 @@ class MultiLabelSparsePrecisionTest(test.TestCase): for class_id in (-1, 1, 3, 8, 10): self._test_sparse_precision_at_k( predictions, labels, k=5, expected=NAN, class_id=class_id) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=NAN, class_id=class_id) def test_3d_no_labels(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] + predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]], + [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]] labels = _binary_3d_label_to_sparse_value( [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) @@ -2083,12 +2170,16 @@ class MultiLabelSparsePrecisionTest(test.TestCase): for class_id in (0, 4, 6, 9): self._test_sparse_precision_at_k( predictions, labels, k=5, expected=0.0, class_id=class_id) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=0.0, class_id=class_id) def test_3d(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] + predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]], + [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]] labels = _binary_3d_label_to_sparse_value( [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) @@ -2096,80 +2187,84 @@ class MultiLabelSparsePrecisionTest(test.TestCase): # Class 2: 4 predictions, all correct. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=4.0 / 4, class_id=2) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=4.0 / 4, class_id=2) # Class 5: 2 predictions, both correct. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=2.0 / 2, class_id=5) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=2.0 / 2, class_id=5) # Class 7: 2 predictions, 1 correct. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=1.0 / 2, class_id=7) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=1.0 / 2, class_id=7) # All classes: 20 predictions, 7 correct. self._test_sparse_precision_at_k( predictions, labels, k=5, expected=7.0 / 20) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=7.0 / 20) def test_3d_ignore_some(self): predictions = [[[0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9], [0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6]], [[0.3, 0.0, 0.7, 0.2, 0.4, 0.9, 0.5, 0.8, 0.1, 0.6], [0.5, 0.1, 0.6, 0.3, 0.8, 0.0, 0.7, 0.2, 0.4, 0.9]]] + predictions_idx = [[[9, 4, 6, 2, 0], [5, 7, 2, 9, 6]], + [[5, 7, 2, 9, 6], [9, 4, 6, 2, 0]]] labels = _binary_3d_label_to_sparse_value( [[[0, 0, 1, 0, 0, 0, 0, 1, 1, 0], [0, 1, 1, 0, 0, 1, 0, 0, 0, 0]], [[0, 1, 1, 0, 0, 1, 0, 1, 0, 0], [0, 0, 1, 0, 0, 0, 0, 0, 1, 0]]]) # Class 2: 2 predictions, both correct. self._test_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=2.0 / 2.0, - class_id=2, + predictions, labels, k=5, expected=2.0 / 2.0, class_id=2, + weights=[[1], [0]]) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=2.0 / 2.0, class_id=2, weights=[[1], [0]]) # Class 2: 2 predictions, both correct. self._test_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=2.0 / 2.0, - class_id=2, + predictions, labels, k=5, expected=2.0 / 2.0, class_id=2, + weights=[[0], [1]]) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=2.0 / 2.0, class_id=2, weights=[[0], [1]]) # Class 7: 1 incorrect prediction. self._test_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=0.0 / 1.0, - class_id=7, + predictions, labels, k=5, expected=0.0 / 1.0, class_id=7, + weights=[[1], [0]]) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=0.0 / 1.0, class_id=7, weights=[[1], [0]]) # Class 7: 1 correct prediction. self._test_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=1.0 / 1.0, - class_id=7, + predictions, labels, k=5, expected=1.0 / 1.0, class_id=7, + weights=[[0], [1]]) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=1.0 / 1.0, class_id=7, weights=[[0], [1]]) # Class 7: no predictions. self._test_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=NAN, - class_id=7, + predictions, labels, k=5, expected=NAN, class_id=7, + weights=[[1, 0], [0, 1]]) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=NAN, class_id=7, weights=[[1, 0], [0, 1]]) # Class 7: 2 predictions, 1 correct. self._test_sparse_precision_at_k( - predictions, - labels, - k=5, - expected=1.0 / 2.0, - class_id=7, + predictions, labels, k=5, expected=1.0 / 2.0, class_id=7, + weights=[[0, 1], [1, 0]]) + self._test_precision_at_top_k( + predictions_idx, labels, k=5, expected=1.0 / 2.0, class_id=7, weights=[[0, 1], [1, 0]]) diff --git a/tensorflow/python/kernel_tests/nth_element_op_test.py b/tensorflow/python/kernel_tests/nth_element_op_test.py new file mode 100644 index 0000000000000000000000000000000000000000..58cd46d2d520790e7e29ab8aea59922b7203ba16 --- /dev/null +++ b/tensorflow/python/kernel_tests/nth_element_op_test.py @@ -0,0 +1,174 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +import tensorflow.python.ops.nn_grad # pylint: disable=unused-import +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradients_impl +from tensorflow.python.platform import test + + +class NthElementTest(test.TestCase): + + def _validateNthElement(self, inputs, dtype, n, reverse, expected_values): + np_expected_values = np.array(expected_values) + with self.test_session(use_gpu=False) as sess: + inputs_op = ops.convert_to_tensor(inputs, dtype=dtype) + values_op = nn_ops.nth_element(inputs_op, n, reverse=reverse) + values = sess.run(values_op) + + self.assertShapeEqual(np_expected_values, values_op) + self.assertAllClose(np_expected_values, values) + + def testExample1(self): + inputs = [2.2, 4.4, 1.1, 5.5, 3.3] + self._validateNthElement(inputs, dtypes.float32, 1, False, 2.2) + self._validateNthElement(inputs, dtypes.float32, 1, True, 4.4) + + def testExample2(self): + inputs = [[2.2, 4.4, 1.1], [5.5, 3.3, 6.6]] + self._validateNthElement(inputs, dtypes.float64, 2, False, [4.4, 6.6]) + self._validateNthElement(inputs, dtypes.float64, 2, True, [1.1, 3.3]) + + def testExample3(self): + inputs = [[[2, 4, 1], [5, -3, 6]], + [[7, 9, -8], [9, 0, 4]]] + self._validateNthElement(inputs, dtypes.int32, 0, False, + [[1, -3], [-8, 0]]) + self._validateNthElement(inputs, dtypes.int64, 0, True, + [[4, 6], [9, 9]]) + + def _testFloatLargeInput(self, input_shape): + inputs = np.random.random_sample(input_shape) + n = np.random.randint(input_shape[-1]) + sort_inputs = np.sort(inputs) + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.float32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.float64, n, True, expected_values) + + def _testIntLargeInput(self, input_shape): + inputs = np.random.randint(-1e3, 1e3, input_shape) + n = np.random.randint(input_shape[-1]) + sort_inputs = np.sort(inputs) + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.int32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.int64, n, True, expected_values) + + def _testLargeInput(self, input_shape): + self._testFloatLargeInput(input_shape) + self._testIntLargeInput(input_shape) + + def testLargeInput(self): + self._testLargeInput([1]) + self._testLargeInput([10]) + self._testLargeInput([5, 10]) + self._testLargeInput([50, 100]) + self._testLargeInput([50, 10000]) + self._testLargeInput([50, 10, 100]) + self._testLargeInput([50, 10, 10, 100]) + + def _testEnumerateN(self, input_shape): + inputs = np.random.random_sample(input_shape) + sort_inputs = np.sort(inputs) + for n in range(input_shape[-1]): + expected_values = sort_inputs[..., n] + self._validateNthElement( + inputs, dtypes.float32, n, False, expected_values) + expected_values = sort_inputs[..., ::-1][..., n] + self._validateNthElement( + inputs, dtypes.float64, n, True, expected_values) + + def testEnumerateN(self): + self._testEnumerateN([1]) + self._testEnumerateN([10]) + self._testEnumerateN([10, 10]) + self._testEnumerateN([10, 10, 10]) + self._testEnumerateN([10, 10, 10, 10]) + + def testInvalidInput(self): + with self.assertRaisesRegexp(ValueError, + "at least rank 1 but is rank 0"): + nn_ops.nth_element(5, 0) + + def testInvalidInputAtEval(self): + with self.test_session(use_gpu=False): + v = array_ops.placeholder(dtype=dtypes.float32) + with self.assertRaisesOpError("Input must be >= 1-D"): + nn_ops.nth_element(v, 0).eval(feed_dict={v: 5.0}) + + def testInvalidN(self): + with self.assertRaisesRegexp(ValueError, + "non-negative but is -1"): + nn_ops.nth_element([5], -1) + with self.assertRaisesRegexp(ValueError, + "scalar but has rank 1"): + nn_ops.nth_element([5, 6, 3], [1]) + + def testInvalidNAtEval(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.test_session(use_gpu=False): + n = array_ops.placeholder(dtypes.int32) + values = nn_ops.nth_element(inputs, n) + with self.assertRaisesOpError("Need n >= 0, got -7"): + values.eval(feed_dict={n: -7}) + + def testNTooLarge(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.assertRaisesRegexp(ValueError, + "must have last dimension > n = 2"): + nn_ops.nth_element(inputs, 2) + + def testNTooLargeAtEval(self): + inputs = [[0.1, 0.2], [0.3, 0.4]] + with self.test_session(use_gpu=False): + n = array_ops.placeholder(dtypes.int32) + values = nn_ops.nth_element(inputs, n) + with self.assertRaisesOpError(r"Input must have at least n\+1 columns"): + values.eval(feed_dict={n: 2}) + + def testGradients(self): + with self.test_session(use_gpu=False) as sess: + inputs = array_ops.placeholder(dtypes.int32, shape=[3, 5]) + values = nn_ops.nth_element(inputs, 3) + grad = sess.run( + gradients_impl.gradients( + values, inputs, grad_ys=[[-1., 2., 5.]]), + feed_dict={inputs: [[2, -1, 1000, 3, 1000], + [1, 5, 2, 4, 3], + [2, 2, 2, 2, 2], + ]}) + self.assertAllClose(grad[0], [[0, 0, -0.5, 0, -0.5], + [0, 0, 0, 2, 0], + [1, 1, 1, 1, 1], + ]) + + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py index ca1f3f878f2f9da763946709299f8341b756c94d..2c766e364073fc8c92156f19d08753367982e7fc 100644 --- a/tensorflow/python/kernel_tests/pad_op_test.py +++ b/tensorflow/python/kernel_tests/pad_op_test.py @@ -193,6 +193,25 @@ class PadOpTest(test.TestCase): with self.assertRaisesRegexp(ValueError, "Unknown padding mode"): array_ops.pad(x, [[1, 0], [2, 1]], mode="weird").eval() + def testPaddingTypes(self): + paddings = [[1, 0], [2, 3], [0, 2]] + inputs = np.random.randint(-100, 100, (4, 4, 3)).astype(np.float32) + for mode in ("CONSTANT", "REFLECT", "SYMMETRIC", "reflect", "symmetric", + "constant"): + for padding_dtype in [dtypes.int32, dtypes.int64]: + np_val = self._npPad(inputs, + paddings, + mode=mode, + constant_values=0) + with self.test_session(use_gpu=True): + tf_val = array_ops.pad(inputs, + constant_op.constant(paddings, padding_dtype), + mode=mode, + constant_values=0) + out = tf_val.eval() + self.assertAllEqual(np_val, out) + self.assertShapeEqual(np_val, tf_val) + def testIntTypes(self): # TODO(touts): Figure out why the padding tests do not work on GPU # for int types and rank > 2. @@ -284,6 +303,15 @@ class PadOpTest(test.TestCase): self.assertAllEqual(inp, out) self.assertShapeEqual(inp, tf_val) + def testPadTypes(self): + for dtype in [dtypes.int32, dtypes.int64]: + paddings = np.zeros((0, 2)) + inp = np.asarray(7) + with self.test_session(use_gpu=True): + tf_val = array_ops.pad(inp, constant_op.constant(paddings, dtype=dtype)) + out = tf_val.eval() + self.assertAllEqual(inp, out) + self.assertShapeEqual(inp, tf_val) if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py index 4bd5b7979722166d11da505046eaa66885d034b8..7ed99c1be9b62a145b9584fd6412f1074f501ae8 100644 --- a/tensorflow/python/kernel_tests/py_func_test.py +++ b/tensorflow/python/kernel_tests/py_func_test.py @@ -148,6 +148,21 @@ class PyOpTest(test.TestCase): z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) self.assertListEqual(list(z.eval()), [b"hello there", b"hi there"]) + def testObjectArraysAreConvertedToBytes(self): + + def read_object_array(): + return np.array([b" there", u" ya"], dtype=np.object) + + def read_and_return_strings(x, y): + return x + y + + with self.test_session(): + x = constant_op.constant(["hello", "hi"], dtypes.string) + y, = script_ops.py_func(read_object_array, [], + [dtypes.string]) + z, = script_ops.py_func(read_and_return_strings, [x, y], [dtypes.string]) + self.assertListEqual(list(z.eval()), [b"hello there", b"hi ya"]) + def testStringPadding(self): correct = [b"this", b"is", b"a", b"test"] with self.test_session(): diff --git a/tensorflow/python/kernel_tests/qr_op_test.py b/tensorflow/python/kernel_tests/qr_op_test.py index b4fd89bd03784082575be592f19792ca18d0b899..8848c15e765236c2ae2817cce1510c4c1ab04740 100644 --- a/tensorflow/python/kernel_tests/qr_op_test.py +++ b/tensorflow/python/kernel_tests/qr_op_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python.framework import constant_op from tensorflow.python.ops import array_ops +from tensorflow.python.ops import gradient_checker from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import random_ops @@ -140,11 +141,11 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_): x_reshape = np.reshape(x_np, (-1, x_np.shape[-2], x_np.shape[-1])) for i in range(new_first_dim): if full_matrices_: - np_q_reshape[i,:,:], _ = \ - np.linalg.qr(x_reshape[i,:,:], mode="complete") + np_q_reshape[i, :, :], _ = np.linalg.qr( + x_reshape[i, :, :], mode="complete") else: - np_q_reshape[i,:,:], _ = \ - np.linalg.qr(x_reshape[i,:,:], mode="reduced") + np_q_reshape[i, :, :], _ = np.linalg.qr( + x_reshape[i, :, :], mode="reduced") np_q = np.reshape(np_q_reshape, q_dims) CompareOrthogonal(self, np_q, q_tf_val, min(shape_[-2:])) CheckApproximation(self, x_np, q_tf_val, r_tf_val) @@ -153,6 +154,46 @@ def _GetQrOpTest(dtype_, shape_, full_matrices_, use_static_shape_): return Test +class QrGradOpTest(test.TestCase): + pass + + +def _GetQrGradOpTest(dtype_, shape_, full_matrices_): + + def Test(self): + np.random.seed(42) + a = np.random.uniform(low=-1.0, high=1.0, size=shape_).astype(dtype_) + if dtype_ in [np.complex64, np.complex128]: + a += 1j * np.random.uniform( + low=-1.0, high=1.0, size=shape_).astype(dtype_) + # Optimal stepsize for central difference is O(epsilon^{1/3}). + epsilon = np.finfo(dtype_).eps + delta = 0.1 * epsilon**(1.0 / 3.0) + if dtype_ in [np.float32, np.complex64]: + tol = 3e-2 + else: + tol = 1e-6 + with self.test_session(use_gpu=True): + tf_a = constant_op.constant(a) + tf_b = linalg_ops.qr(tf_a, full_matrices=full_matrices_) + for b in tf_b: + x_init = np.random.uniform( + low=-1.0, high=1.0, size=shape_).astype(dtype_) + if dtype_ in [np.complex64, np.complex128]: + x_init += 1j * np.random.uniform( + low=-1.0, high=1.0, size=shape_).astype(dtype_) + theoretical, numerical = gradient_checker.compute_gradient( + tf_a, + tf_a.get_shape().as_list(), + b, + b.get_shape().as_list(), + x_init_value=x_init, + delta=delta) + self.assertAllClose(theoretical, numerical, atol=tol, rtol=tol) + + return Test + + if __name__ == "__main__": for dtype in np.float32, np.float64, np.complex64, np.complex128: for rows in 1, 2, 5, 10, 32, 100: @@ -168,4 +209,21 @@ if __name__ == "__main__": _AddTest(QrOpTest, "Qr", name, _GetQrOpTest(dtype, shape, full_matrices, use_static_shape)) + + # TODO(pfau): Get working with complex types. + # TODO(pfau): Get working with full_matrices when rows != cols + # TODO(pfau): Get working when rows < cols + # TODO(pfau): Get working with shapeholders (dynamic shapes) + for full_matrices in False, True: + for dtype in np.float32, np.float64: + for rows in 1, 2, 5, 10: + for cols in 1, 2, 5, 10: + if rows == cols or (not full_matrices and rows > cols): + for batch_dims in [(), (3,)] + [(3, 2)] * (max(rows, cols) < 10): + shape = batch_dims + (rows, cols) + name = "%s_%s_full_%s" % (dtype.__name__, + "_".join(map(str, shape)), + full_matrices) + _AddTest(QrGradOpTest, "QrGrad", name, + _GetQrGradOpTest(dtype, shape, full_matrices)) test.main() diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..88a4ddf7f29ec772282e7a8e2b59f144f1a968c2 --- /dev/null +++ b/tensorflow/python/kernel_tests/random/BUILD @@ -0,0 +1,135 @@ +# Tests of TensorFlow kernels written using the Python API. + +package( + default_visibility = ["//tensorflow:internal"], +) + +licenses(["notice"]) # Apache 2.0 + +load("//tensorflow:tensorflow.bzl", "tf_py_test") +load("//tensorflow:tensorflow.bzl", "cuda_py_test") +load("//tensorflow:tensorflow.bzl", "sycl_py_test") + +# CPU only tests should use tf_py_test, GPU tests use cuda_py_test +# Please avoid the py_tests and cuda_py_tests (plural) while we +# fix the shared/overbroad dependencies. + +tf_py_test( + name = "random_shuffle_queue_test", + size = "small", + srcs = ["random_shuffle_queue_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:data_flow_ops", + "//tensorflow/python:errors", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", + ], +) + +cuda_py_test( + name = "multinomial_op_test", + size = "small", + srcs = ["multinomial_op_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:random_ops", + ], +) + +cuda_py_test( + name = "multinomial_op_big_test", + size = "medium", + srcs = ["multinomial_op_big_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/core:protos_all_py", + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:control_flow_ops", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:math_ops", + "//tensorflow/python:random_ops", + ], + shard_count = 3, +) + +cuda_py_test( + name = "random_crop_test", + size = "small", + srcs = ["random_crop_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:client_testlib", + "//tensorflow/python:random_ops", + ], +) + +cuda_py_test( + name = "random_ops_test", + size = "medium", + srcs = ["random_ops_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:random_ops", + ], +) + +cuda_py_test( + name = "random_gamma_test", + size = "medium", + srcs = ["random_gamma_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", + "//tensorflow/python:random_ops", + ], + shard_count = 4, + tags = ["nozapfhahn"], +) + +cuda_py_test( + name = "random_poisson_test", + size = "medium", + srcs = ["random_poisson_test.py"], + additional_deps = [ + "//third_party/py/numpy", + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:platform", + "//tensorflow/python:random_ops", + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/python/kernel_tests/multinomial_op_big_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_big_test.py similarity index 100% rename from tensorflow/python/kernel_tests/multinomial_op_big_test.py rename to tensorflow/python/kernel_tests/random/multinomial_op_big_test.py diff --git a/tensorflow/python/kernel_tests/multinomial_op_test.py b/tensorflow/python/kernel_tests/random/multinomial_op_test.py similarity index 100% rename from tensorflow/python/kernel_tests/multinomial_op_test.py rename to tensorflow/python/kernel_tests/random/multinomial_op_test.py diff --git a/tensorflow/python/kernel_tests/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_crop_test.py rename to tensorflow/python/kernel_tests/random/random_crop_test.py diff --git a/tensorflow/python/kernel_tests/random_gamma_test.py b/tensorflow/python/kernel_tests/random/random_gamma_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_gamma_test.py rename to tensorflow/python/kernel_tests/random/random_gamma_test.py diff --git a/tensorflow/python/kernel_tests/random_ops_test.py b/tensorflow/python/kernel_tests/random/random_ops_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_ops_test.py rename to tensorflow/python/kernel_tests/random/random_ops_test.py diff --git a/tensorflow/python/kernel_tests/random_poisson_test.py b/tensorflow/python/kernel_tests/random/random_poisson_test.py similarity index 100% rename from tensorflow/python/kernel_tests/random_poisson_test.py rename to tensorflow/python/kernel_tests/random/random_poisson_test.py diff --git a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py similarity index 99% rename from tensorflow/python/kernel_tests/random_shuffle_queue_test.py rename to tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py index 1b84af68230ec69950a608660ade9f7fc1271868..c4e16ff6280cc7ce121955474fe8ec45acd57f95 100644 --- a/tensorflow/python/kernel_tests/random_shuffle_queue_test.py +++ b/tensorflow/python/kernel_tests/random/random_shuffle_queue_test.py @@ -654,7 +654,8 @@ class RandomShuffleQueueTest(test.TestCase): def testBlockingDequeueFromClosedQueue(self): with self.test_session() as sess: - q = data_flow_ops.RandomShuffleQueue(10, 2, dtypes_lib.float32) + min_size = 2 + q = data_flow_ops.RandomShuffleQueue(10, min_size, dtypes_lib.float32) elems = [10.0, 20.0, 30.0, 40.0] enqueue_op = q.enqueue_many((elems,)) close_op = q.close() @@ -664,20 +665,24 @@ class RandomShuffleQueueTest(test.TestCase): results = [] - def dequeue(): - for _ in elems: - results.append(sess.run(dequeued_t)) + # Manually dequeue until we hit min_size. + results.append(sess.run(dequeued_t)) + results.append(sess.run(dequeued_t)) + + def blocking_dequeue(): + results.append(sess.run(dequeued_t)) + results.append(sess.run(dequeued_t)) + self.assertItemsEqual(elems, results) # Expect the operation to fail due to the queue being closed. with self.assertRaisesRegexp(errors_impl.OutOfRangeError, "is closed and has insufficient"): sess.run(dequeued_t) - dequeue_thread = self.checkedThread(target=dequeue) + dequeue_thread = self.checkedThread(target=blocking_dequeue) dequeue_thread.start() - # The close_op should run after the dequeue_thread has blocked. - # TODO(mrry): Figure out how to do this without sleeping. time.sleep(0.1) + # The dequeue thread blocked when it hit the min_size requirement. self.assertEqual(len(results), 2) close_op.run() diff --git a/tensorflow/python/kernel_tests/range_dataset_op_test.py b/tensorflow/python/kernel_tests/range_dataset_op_test.py index 8291967155656e5fbab82f33cff5f3e760c9acac..0c530522b8316e3c17716ad43c595b4af754e39c 100644 --- a/tensorflow/python/kernel_tests/range_dataset_op_test.py +++ b/tensorflow/python/kernel_tests/range_dataset_op_test.py @@ -27,6 +27,8 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops +from tensorflow.python.ops import parsing_ops from tensorflow.python.ops import variables from tensorflow.python.platform import gfile from tensorflow.python.platform import test @@ -169,6 +171,21 @@ class RangeDatasetTest(test.TestCase): def _iterator_checkpoint_prefix(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_prefix(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_prefix()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def testSaveRestore(self): def _build_graph(start, stop): @@ -176,10 +193,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -222,14 +237,13 @@ class RangeDatasetTest(test.TestCase): def testRestoreWithoutBuildingDatasetGraph(self): - def _build_graph(start, stop, num_epochs, path): + def _build_graph(start, stop, num_epochs): dataset = dataset_ops.Dataset.range(start, stop).repeat(num_epochs) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -238,10 +252,8 @@ class RangeDatasetTest(test.TestCase): num_epochs = 5 break_point = 5 break_epoch = 3 - path = self._iterator_checkpoint_prefix() with ops.Graph().as_default() as g: - init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs, - path) + init_op, get_next, save_op, _ = _build_graph(start, stop, num_epochs) with self.test_session(graph=g) as sess: sess.run(variables.global_variables_initializer()) sess.run(init_op) @@ -258,8 +270,7 @@ class RangeDatasetTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + restore_op = self._restore_op(iterator._iterator_resource) get_next = iterator.get_next() with self.test_session(graph=g) as sess: sess.run(restore_op) @@ -278,10 +289,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -319,10 +328,8 @@ class RangeDatasetTest(test.TestCase): iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op # Saving and restoring in different sessions. @@ -355,10 +362,8 @@ class RangeDatasetTest(test.TestCase): stop).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -400,10 +405,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 @@ -447,10 +450,8 @@ class RangeDatasetTest(test.TestCase): start, stop).repeat(num_epochs).make_initializable_iterator() init_op = iterator.initializer get_next = iterator.get_next() - path = self._iterator_checkpoint_prefix() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next, save_op, restore_op start = 2 diff --git a/tensorflow/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/python/kernel_tests/reader_dataset_ops_test.py index 38420328efec9f6489898dfa8f485ec2395ca03f..c8e7333b4b9949b6b6ef5f7f6d63e5ff8c354c37 100644 --- a/tensorflow/python/kernel_tests/reader_dataset_ops_test.py +++ b/tensorflow/python/kernel_tests/reader_dataset_ops_test.py @@ -31,6 +31,8 @@ from tensorflow.python.framework import tensor_shape from tensorflow.python.lib.io import python_io from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_dataset_ops +from tensorflow.python.ops import io_ops +from tensorflow.python.ops import parsing_ops from tensorflow.python.platform import test from tensorflow.python.util import compat @@ -273,18 +275,31 @@ class FixedLengthRecordReaderTest(test.TestCase): def _iterator_checkpoint_path(self): return os.path.join(self.get_temp_dir(), "iterator") + def _save_op(self, iterator_resource): + iterator_state_variant = gen_dataset_ops.serialize_iterator( + iterator_resource) + save_op = io_ops.write_file( + self._iterator_checkpoint_path(), + parsing_ops.serialize_tensor(iterator_state_variant)) + return save_op + + def _restore_op(self, iterator_resource): + iterator_state_variant = parsing_ops.parse_tensor( + io_ops.read_file(self._iterator_checkpoint_path()), dtypes.variant) + restore_op = gen_dataset_ops.deserialize_iterator(iterator_resource, + iterator_state_variant) + return restore_op + def _build_iterator_graph(self, num_epochs): filenames = self._createFiles() - path = self._iterator_checkpoint_path() dataset = (readers.FixedLengthRecordDataset( filenames, self._record_bytes, self._header_bytes, self._footer_bytes) .repeat(num_epochs)) iterator = dataset.make_initializable_iterator() init_op = iterator.initializer get_next_op = iterator.get_next() - save_op = gen_dataset_ops.save_iterator(iterator._iterator_resource, path) - restore_op = gen_dataset_ops.restore_iterator(iterator._iterator_resource, - path) + save_op = self._save_op(iterator._iterator_resource) + restore_op = self._restore_op(iterator._iterator_resource) return init_op, get_next_op, save_op, restore_op def _restore_iterator(self): @@ -292,8 +307,7 @@ class FixedLengthRecordReaderTest(test.TestCase): output_shapes = tensor_shape.scalar() iterator = iterator_ops.Iterator.from_structure(output_types, output_shapes) get_next = iterator.get_next() - restore_op = gen_dataset_ops.restore_iterator( - iterator._iterator_resource, self._iterator_checkpoint_path()) + restore_op = self._restore_op(iterator._iterator_resource) return restore_op, get_next def testSaveRestore(self): diff --git a/tensorflow/python/kernel_tests/reduction_ops_test.py b/tensorflow/python/kernel_tests/reduction_ops_test.py index c794351fe99fe182eb3592ccbddf0ff2103d4f5b..2dc65b13849439b413b39c7dfec6e86225f6c49b 100644 --- a/tensorflow/python/kernel_tests/reduction_ops_test.py +++ b/tensorflow/python/kernel_tests/reduction_ops_test.py @@ -163,6 +163,13 @@ class SumReductionTest(BaseReductionTest): reduction_axes = tuple(reduction_axes) return np.sum(x, axis=reduction_axes, keepdims=keep_dims) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_sum([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -193,6 +200,7 @@ class SumReductionTest(BaseReductionTest): tf_out_mean = sess.run(tf_mean) self.assertAllClose(tf_out_mean, 1.) + def testFloat32(self): for rank in range(1, _MAX_RANK + 1): np_arr = self._makeIncremental((2,) * rank, dtypes.float32) @@ -369,6 +377,13 @@ class MeanReductionTest(BaseReductionTest): return np_sum // count return np_sum / count + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_mean([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -435,6 +450,13 @@ class ProdReductionTest(BaseReductionTest): reduction_axes = tuple(reduction_axes) return np.prod(x, axis=reduction_axes, keepdims=keep_dims) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_prod([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -531,6 +553,13 @@ class MinReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_min([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -637,6 +666,13 @@ class MaxReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_max([0, 0], constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, 0) + def testInfinity(self): for dtype in [np.float32, np.float64]: for special_value_x in [-np.inf, np.inf]: @@ -757,6 +793,14 @@ class AllReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_all([True, True], + constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, True) + def testAll3D(self): # Create a 3D array of bools and reduce across all possible # dimensions @@ -798,6 +842,14 @@ class AnyReductionTest(test.TestCase): self._compare(x, reduction_axes, True, use_gpu=True) self._compare(x, reduction_axes, True, use_gpu=False) + def testAxesType(self): + for dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True) as sess: + v = math_ops.reduce_any([True, True], + constant_op.constant(0, dtype=dtype)) + tf_v = sess.run(v) + self.assertAllEqual(tf_v, True) + def testAll3D(self): # Create a 3D array of bools and reduce across all possible # dimensions diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py index 6f2bc2f7521be8462c3389bd06fc966a52df79b0..8f328cea631767085177d3e555c4f7565abc2c27 100644 --- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py +++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py @@ -17,6 +17,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gc + import numpy as np from tensorflow.python.eager import context @@ -30,6 +32,7 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops +from tensorflow.python.ops import state_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -37,6 +40,12 @@ from tensorflow.python.platform import test class ResourceVariableOpsTest(test_util.TensorFlowTestCase): + def tearDown(self): + gc.collect() + # This will only contain uncollectable garbage, i.e. reference cycles + # involving objects with __del__ defined. + self.assertEqual(0, len(gc.garbage)) + def testHandleDtypeShapeMatch(self): with self.test_session(): handle = resource_variable_ops.var_handle_op(dtype=dtypes.int32, shape=[]) @@ -53,6 +62,18 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): 0, dtype=dtypes.int32)).run() + def testEagerNameNotIdentity(self): + with context.eager_mode(): + v0 = resource_variable_ops.ResourceVariable(1.0, name="a") + v1 = resource_variable_ops.ResourceVariable(2.0, name="a") + self.assertAllEqual(v0.numpy(), 1.0) + self.assertAllEqual(v1.numpy(), 2.0) + + def testEagerNameNotNeeded(self): + with context.eager_mode(): + v0 = resource_variable_ops.ResourceVariable(1.0) + self.assertAllEqual(v0.numpy(), 1.0) + def testReadVariableDtypeMismatchEager(self): with context.eager_mode(): handle = resource_variable_ops.var_handle_op( @@ -62,6 +83,17 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): "Expected float got int32."): _ = resource_variable_ops.read_variable_op(handle, dtype=dtypes.float32) + def testEagerInitializedValue(self): + with context.eager_mode(): + variable = resource_variable_ops.ResourceVariable(1.0, name="eager-init") + self.assertAllEqual(variable.numpy(), 1.0) + self.assertAllEqual(variable.initialized_value().numpy(), 1.0) + + def testEagerBool(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable(False, name="bool_test") + self.assertAllEqual(bool(v), False) + def testAssignVariableDtypeMismatchEager(self): with context.eager_mode(): handle = resource_variable_ops.var_handle_op( @@ -172,10 +204,30 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): self.assertEqual(v.handle.op.colocation_groups(), v.initializer.inputs[1].op.colocation_groups()) + def testHandleNumpy(self): + with context.eager_mode(): + with self.assertRaises(ValueError): + resource_variable_ops.ResourceVariable( + 1.0, name="handle-numpy").handle.numpy() + + def testCountUpTo(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable(0, name="upto") + self.assertAllEqual(v.count_up_to(1), 0) + with self.assertRaises(errors.OutOfRangeError): + v.count_up_to(1) + + def testCountUpToFunction(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable(0, name="upto") + self.assertAllEqual(state_ops.count_up_to(v, 1), 0) + with self.assertRaises(errors.OutOfRangeError): + state_ops.count_up_to(v, 1) + @test_util.run_in_graph_and_eager_modes() def testInitFnDtype(self): v = resource_variable_ops.ResourceVariable( - initial_value=lambda: 1, dtype=dtypes.float32) + initial_value=lambda: 1, dtype=dtypes.float32, name="var0") self.assertEqual(dtypes.float32, v.value().dtype) @test_util.run_in_graph_and_eager_modes() @@ -186,26 +238,27 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testInitializeAllVariables(self): - v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.float32) + v = resource_variable_ops.ResourceVariable(1, dtype=dtypes.float32, + name="var0") self.evaluate(variables.global_variables_initializer()) self.assertEqual(1.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testOperatorOverload(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.assertEqual(2.0, self.evaluate(v + v)) @test_util.run_in_graph_and_eager_modes() def testAssignMethod(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.evaluate(v.assign(2.0)) self.assertEqual(2.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testLoad(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) v.load(2.0) self.assertEqual(2.0, self.evaluate(v.value())) @@ -231,21 +284,21 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): @test_util.run_in_graph_and_eager_modes() def testAssignAddMethod(self): - v = resource_variable_ops.ResourceVariable(1.0) + v = resource_variable_ops.ResourceVariable(1.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.evaluate(v.assign_add(1.0)) self.assertEqual(2.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testAssignSubMethod(self): - v = resource_variable_ops.ResourceVariable(3.0) + v = resource_variable_ops.ResourceVariable(3.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.evaluate(v.assign_sub(1.0)) self.assertEqual(2.0, self.evaluate(v.value())) @test_util.run_in_graph_and_eager_modes() def testDestroyResource(self): - v = resource_variable_ops.ResourceVariable(3.0) + v = resource_variable_ops.ResourceVariable(3.0, name="var0") self.evaluate(variables.global_variables_initializer()) self.assertEqual(3.0, self.evaluate(v.value())) self.evaluate(resource_variable_ops.destroy_resource_op(v.handle)) @@ -272,8 +325,10 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with variable_scope.variable_scope("foo"): var = variable_scope.get_variable("x", shape=[1, 1], dtype=dtypes.float32) - assign = var.assign(np.zeros(shape=[2, 2])) - self.evaluate(assign) + with self.assertRaisesRegexp(ValueError, + "Shapes.*and.*are incompatible"): + assign = var.assign(np.zeros(shape=[2, 2])) + self.evaluate(assign) def testDtypeAfterFromProto(self): v = resource_variable_ops.ResourceVariable(2.0) @@ -297,34 +352,38 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with self.assertRaisesRegexp(ValueError, "No attr named '_class'"): _ = w.value().op.get_attr("_class") - @test_util.run_in_graph_and_eager_modes() def testSharedName(self): - v = resource_variable_ops.ResourceVariable(300.0, name="var4") - self.evaluate(variables.global_variables_initializer()) + with self.test_session(): + v = resource_variable_ops.ResourceVariable(300.0, name="var4") + variables.global_variables_initializer().run() - w = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4") - w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) - self.assertEqual(300.0, self.evaluate(w_read)) + w = resource_variable_ops.var_handle_op( + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var4", + # Needed in Eager since we get a unique container name by default. + container=ops.get_default_graph()._container) + w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) + self.assertEqual(300.0, w_read.eval()) - x = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5") - with self.assertRaisesOpError("Resource .*/var5/.* does not exist"): - x_read = resource_variable_ops.read_variable_op(x, v.dtype.base_dtype) - self.evaluate(x_read) + x = resource_variable_ops.var_handle_op( + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="var5", + container=ops.get_default_graph()._container) + with self.assertRaisesOpError("Resource .*/var5/.* does not exist"): + resource_variable_ops.read_variable_op(x, v.dtype.base_dtype).eval() - @test_util.run_in_graph_and_eager_modes() def testSharedNameWithNamescope(self): - with ops.name_scope("foo"): - v = resource_variable_ops.ResourceVariable(300.0, name="var6") - self.assertEqual("foo/var6", v._shared_name) # pylint: disable=protected-access - self.assertEqual("foo/var6:0", v.name) - self.evaluate(variables.global_variables_initializer()) - - w = resource_variable_ops.var_handle_op( - dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6") - w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) - self.assertEqual(300.0, self.evaluate(w_read)) + with self.test_session(): + with ops.name_scope("foo"): + v = resource_variable_ops.ResourceVariable(300.0, name="var6") + self.assertEqual("foo/var6", v._shared_name) # pylint: disable=protected-access + self.assertEqual("foo/var6:0", v.name) + self.evaluate(variables.global_variables_initializer()) + + w = resource_variable_ops.var_handle_op( + dtype=v.dtype.base_dtype, shape=v.get_shape(), shared_name="foo/var6", + # Needed in Eager since we get a unique container name by default. + container=ops.get_default_graph()._container) + w_read = resource_variable_ops.read_variable_op(w, v.dtype.base_dtype) + self.assertEqual(300.0, self.evaluate(w_read)) @test_util.run_in_graph_and_eager_modes() def testShape(self): @@ -426,12 +485,19 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase): with context.eager_mode(): var = resource_variable_ops.ResourceVariable(initial_value=1.0, name="var8") - var.__del__() + var_handle = var._handle + del var with self.assertRaisesRegexp(errors.NotFoundError, - r"Resource .*\/var8\/.* does not exist."): - resource_variable_ops.destroy_resource_op(var._handle, + r"Resource .* does not exist."): + resource_variable_ops.destroy_resource_op(var_handle, ignore_lookup_error=False) + def testScatterUpdate(self): + with context.eager_mode(): + v = resource_variable_ops.ResourceVariable([1.0, 2.0], name="update") + state_ops.scatter_update(v, [1], [3.0]) + self.assertAllEqual([1.0, 3.0], v.numpy()) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/kernel_tests/rnn_test.py b/tensorflow/python/kernel_tests/rnn_test.py index a644e6a44fa7f0e1bec7e3ea664a8a79a202ad05..d8f4b439e37981f3d21181feae9baa8d492ee1d5 100644 --- a/tensorflow/python/kernel_tests/rnn_test.py +++ b/tensorflow/python/kernel_tests/rnn_test.py @@ -26,9 +26,12 @@ import numpy as np from tensorflow.contrib import rnn as contrib_rnn from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session +from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops as ops_lib from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gradients_impl @@ -82,9 +85,13 @@ class RNNTest(test.TestCase): self._seed = 23489 np.random.seed(self._seed) + @test_util.run_in_graph_and_eager_modes() def testInvalidSequenceLengthShape(self): cell = Plus1RNNCell() - inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))] + if context.in_graph_mode(): + inputs = [array_ops.placeholder(dtypes.float32, shape=(3, 4))] + else: + inputs = [constant_op.constant(np.ones((3, 4)))] with self.assertRaisesRegexp(ValueError, "must be a vector"): rnn.dynamic_rnn( cell, @@ -92,45 +99,77 @@ class RNNTest(test.TestCase): dtype=dtypes.float32, sequence_length=[[4]]) + @test_util.run_in_graph_and_eager_modes() def testBatchSizeFromInput(self): cell = Plus1RNNCell() + in_graph_mode = context.in_graph_mode() # With static batch size - inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) + if in_graph_mode: + inputs = array_ops.placeholder(dtypes.float32, shape=(3, 4, 5)) + initial_state = array_ops.placeholder(dtypes.float32, shape=(3, 5)) + else: + inputs = np.zeros((3, 4, 5), dtype=np.float32) + initial_state = np.zeros((3, 5), dtype=np.float32) + # - Without initial_state outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) - self.assertEqual(3, outputs.shape[0].value) - self.assertEqual(3, state.shape[0].value) + if in_graph_mode: + self.assertEqual(3, outputs.shape[0].value) + self.assertEqual(3, state.shape[0].value) + else: + self.assertEqual(3, outputs.shape[0]) + self.assertEqual(3, state.shape[0]) + # - With initial_state outputs, state = rnn.dynamic_rnn( - cell, - inputs, - initial_state=array_ops.placeholder(dtypes.float32, shape=(3, 5))) - self.assertEqual(3, outputs.shape[0].value) - self.assertEqual(3, state.shape[0].value) + cell, inputs, initial_state=initial_state) + if in_graph_mode: + self.assertEqual(3, outputs.shape[0].value) + self.assertEqual(3, state.shape[0].value) + else: + self.assertEqual(3, outputs.shape[0]) + self.assertEqual(3, state.shape[0]) + # Without static batch size - inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) - # - Without initial_state - outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) - self.assertEqual(None, outputs.shape[0].value) - self.assertEqual(None, state.shape[0].value) - # - With initial_state - outputs, state = rnn.dynamic_rnn( - cell, - inputs, - initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) - self.assertEqual(None, outputs.shape[0].value) - self.assertEqual(None, state.shape[0].value) + # Tensor shapes are fully determined in Eager mode, so only run this + # test in graph mode. + if in_graph_mode: + inputs = array_ops.placeholder(dtypes.float32, shape=(None, 4, 5)) + # - Without initial_state + outputs, state = rnn.dynamic_rnn(cell, inputs, dtype=dtypes.float32) + self.assertEqual(None, outputs.shape[0].value) + self.assertEqual(None, state.shape[0].value) + # - With initial_state + outputs, state = rnn.dynamic_rnn( + cell, + inputs, + initial_state=array_ops.placeholder(dtypes.float32, shape=(None, 5))) + self.assertEqual(None, outputs.shape[0].value) + self.assertEqual(None, state.shape[0].value) + @test_util.run_in_graph_and_eager_modes() def testScalarStateIsAccepted(self): cell = ScalarStateRNNCell() - inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1)) + in_graph_mode = context.in_graph_mode() + + if in_graph_mode: + inputs = array_ops.placeholder(dtypes.float32, shape=(1, 4, 1)) + else: + inputs = np.array([[[1], [2], [3], [4]]], dtype=np.float32) + with self.test_session() as sess: outputs, state = rnn.dynamic_rnn( cell, inputs, dtype=dtypes.float32, sequence_length=[4]) - outputs, state = sess.run( - [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]}) - self.assertAllEqual(outputs, [[[1], [2], [3], [4]]]) - self.assertEqual(state, 4) + if in_graph_mode: + outputs, state = sess.run( + [outputs, state], feed_dict={inputs: [[[1], [2], [3], [4]]]}) + + if in_graph_mode: + self.assertAllEqual(outputs, np.array([[[1], [2], [3], [4]]])) + self.assertEqual(state, 4) + else: + self.assertAllEqual(outputs.numpy(), np.array([[[1], [2], [3], [4]]])) + self.assertEqual(state.numpy(), 4) ######### Benchmarking RNN code diff --git a/tensorflow/python/kernel_tests/scan_ops_test.py b/tensorflow/python/kernel_tests/scan_ops_test.py index 6b2b589a06120d099bc6eaf438a0dcb9ff8e2295..08b4a2aaae2469a7fedb13d47493c02cf8306a9b 100644 --- a/tensorflow/python/kernel_tests/scan_ops_test.py +++ b/tensorflow/python/kernel_tests/scan_ops_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import numpy as np +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import gradient_checker @@ -92,6 +94,14 @@ class CumsumTest(test.TestCase): for axis in (-1, 0): self._compareAll(x, axis) + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True): + axis = constant_op.constant(0, axis_dtype) + tf_out = math_ops.cumsum(x, axis).eval() + def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) @@ -190,6 +200,14 @@ class CumprodTest(test.TestCase): for axis in (-1, 0): self._compareAll(x, axis) + def testAxisType(self): + for dtype in self.valid_dtypes: + x = np.arange(1, 6).reshape([5]).astype(dtype) + for axis_dtype in [dtypes.int64, dtypes.int32]: + with self.test_session(use_gpu=True): + axis = constant_op.constant(0, axis_dtype) + tf_out = math_ops.cumprod(x, axis).eval() + def test1D(self): for dtype in self.valid_dtypes: x = np.arange(1, 6).reshape([5]).astype(dtype) diff --git a/tensorflow/python/kernel_tests/shape_ops_test.py b/tensorflow/python/kernel_tests/shape_ops_test.py index 52cf904528b27dee20679f044d92c84b49bef53b..a9fc699b21e883db6c627c478ad29c79475b1271 100644 --- a/tensorflow/python/kernel_tests/shape_ops_test.py +++ b/tensorflow/python/kernel_tests/shape_ops_test.py @@ -411,14 +411,16 @@ class TileTest(test.TestCase): self.assertEqual(7, result) def testSimple(self): - with self.test_session(): - inp = np.random.rand(4, 1).astype(np.float32) - a = constant_op.constant(inp) - tiled = array_ops.tile(a, [1, 4]) - result = tiled.eval() - self.assertEqual(result.shape, (4, 4)) - self.assertEqual([4, 4], tiled.get_shape()) - self.assertTrue((result == np.tile(inp, (1, 4))).all()) + # multiples could be int32 or int64 + for dtype in [dtypes.int32, dtypes.int64]: + with self.test_session(use_gpu=True): + inp = np.random.rand(4, 1).astype(np.float32) + a = constant_op.constant(inp) + tiled = array_ops.tile(a, constant_op.constant([1, 4], dtype=dtype)) + result = tiled.eval() + self.assertEqual(result.shape, (4, 4)) + self.assertEqual([4, 4], tiled.get_shape()) + self.assertTrue((result == np.tile(inp, (1, 4))).all()) def testIdentityTileAndGrad(self): with self.test_session(): diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py index 53e045fe86b73fa6a107650caa22956fc55dc9cf..a1fc6d63d454c6130874da948a2c84a3e4384b20 100644 --- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py +++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py @@ -22,17 +22,22 @@ import numpy as np from tensorflow.core.protobuf import config_pb2 from tensorflow.python.client import session as session_lib +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_data_flow_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import init_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_grad from tensorflow.python.ops import tensor_array_ops +from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables import tensorflow.python.ops.nn_grad # pylint: disable=unused-import from tensorflow.python.platform import test @@ -51,6 +56,11 @@ def _make_converter(tf_dtype): return _converter +def _make_ta(size, name, dtype=dtypes.float32, infer_shape=False): + return tensor_array_ops.TensorArray( + dtype=dtype, tensor_array_name=name, size=size, infer_shape=infer_shape) + + class TensorArrayTest(test.TestCase): @classmethod @@ -63,8 +73,9 @@ class TensorArrayTest(test.TestCase): super(TensorArrayTest, cls).tearDownClass() session_lib.Session.reset(cls._workers[0].target) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteRead(self): - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", @@ -79,7 +90,7 @@ class TensorArrayTest(test.TestCase): r1 = w2.read(1) r2 = w2.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual([[4.0, 5.0]], d0) self.assertAllEqual([[1.0]], d1) self.assertAllEqual(-3.0, d2) @@ -97,8 +108,9 @@ class TensorArrayTest(test.TestCase): c0 = w2.stack() + c0 = self.evaluate(c0) self.assertAllEqual( - convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0.eval()) + convert([[[4.0, 5.0]], [[6.0, 7.0]], [[8.0, 9.0]]]), c0) def _testTensorArrayWritePackMaybeLegacy(self): self._testTensorArrayWritePack(dtypes.float32) @@ -109,9 +121,11 @@ class TensorArrayTest(test.TestCase): self._testTensorArrayWritePack(dtypes.complex128) self._testTensorArrayWritePack(dtypes.string) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWritePack(self): self._testTensorArrayWritePackMaybeLegacy() + @test_util.run_in_graph_and_eager_modes() def testEmptyTensorArrayPack(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -124,7 +138,8 @@ class TensorArrayTest(test.TestCase): c0 = w2.stack() - self.assertAllEqual([3, 0, 1], c0.eval().shape) + c0 = self.evaluate(c0) + self.assertAllEqual([3, 0, 1], c0.shape) def _testTensorArrayWriteConcat(self, tf_dtype): with self.test_session(use_gpu=True): @@ -139,10 +154,12 @@ class TensorArrayTest(test.TestCase): c0 = w2.concat() + c0 = self.evaluate(c0) self.assertAllEqual( convert([[4.0, 5.0], [104.0, 105.0], [204.0, 205.0], [6.0, 7.0], - [106.0, 107.0], [8.0, 9.0]]), c0.eval()) + [106.0, 107.0], [8.0, 9.0]]), c0) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteConcat(self): self._testTensorArrayWriteConcat(dtypes.float32) self._testTensorArrayWriteConcat(dtypes.float64) @@ -159,55 +176,46 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError("Could not read from TensorArray index 1 " "because it has not yet been written to."): - ta.write(0, [[4.0, 5.0]]).stack().eval() + self.evaluate(ta.write(0, [[4.0, 5.0]]).stack()) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayPackNotAllValuesAvailableFails(self): self._testTensorArrayPackNotAllValuesAvailableFails() def _testTensorArrayUnpackRead(self, tf_dtype): - with self.test_session(use_gpu=True) as session: - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3) - + with self.test_session(use_gpu=True): convert = _make_converter(tf_dtype) + ta = _make_ta(3, "foo", dtype=tf_dtype) # Unpack a vector into scalars w0 = ta.unstack(convert([1.0, 2.0, 3.0])) r0 = w0.read(0) r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert(1.0), d0) self.assertAllEqual(convert(2.0), d1) self.assertAllEqual(convert(3.0), d2) - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3) - # Unpack a matrix into vectors w1 = ta.unstack(convert([[1.0, 1.1], [2.0, 2.1], [3.0, 3.1]])) r0 = w1.read(0) r1 = w1.read(1) r2 = w1.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([1.0, 1.1]), d0) self.assertAllEqual(convert([2.0, 2.1]), d1) self.assertAllEqual(convert([3.0, 3.1]), d2) - # Reset ta because we're going to change the shape, else shape - # inference will throw an error. - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3) - # Try unpacking an empty matrix, which should not cause an error. w2 = ta.unstack(convert([[], [], []])) r0 = w2.read(0) r1 = w2.read(1) r2 = w2.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([]), d0) self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([]), d2) @@ -221,24 +229,23 @@ class TensorArrayTest(test.TestCase): self._testTensorArrayUnpackRead(dtypes.complex128) self._testTensorArrayUnpackRead(dtypes.string) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayUnpackRead(self): self._testTensorArrayUnpackReadMaybeLegacy() def _testTensorArraySplitRead(self, tf_dtype): - with self.test_session(use_gpu=True) as session: - ta = tensor_array_ops.TensorArray( - dtype=tf_dtype, tensor_array_name="foo", size=3, infer_shape=False) - + with self.test_session(use_gpu=True): convert = _make_converter(tf_dtype) # Split an empty vector + ta = _make_ta(3, "foo", dtype=tf_dtype) lengths = constant_op.constant([0, 0, 0]) w0 = ta.split(convert([]), lengths=lengths) r0 = w0.read(0) r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([]), d0) self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([]), d2) @@ -250,7 +257,7 @@ class TensorArrayTest(test.TestCase): r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([1.0, 2.0]), d0) self.assertAllEqual(convert([]), d1) self.assertAllEqual(convert([3.0]), d2) @@ -263,11 +270,12 @@ class TensorArrayTest(test.TestCase): r1 = w0.read(1) r2 = w0.read(2) - d0, d1, d2 = session.run([r0, r1, r2]) + d0, d1, d2 = self.evaluate([r0, r1, r2]) self.assertAllEqual(convert([[1.0, 101.0], [2.0, 201.0]]), d0) self.assertAllEqual(convert([]).reshape(0, 2), d1) self.assertAllEqual(convert([[3.0, 301.0]]), d2) + @test_util.run_in_graph_and_eager_modes() def testTensorArraySplitRead(self): self._testTensorArraySplitRead(dtypes.float32) self._testTensorArraySplitRead(dtypes.float64) @@ -367,59 +375,76 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual(t_g_ta_0, t_g_ta_1) self.assertAllEqual([[4.0, 5.0]], d_r1_0) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteWrongIndexOrDataTypeFails(self): with self.test_session(use_gpu=True): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, tensor_array_name="foo", size=3) - + ta = _make_ta(3, "foo", dtype=dtypes.float32) + in_graph_mode = context.in_graph_mode() # Test writing the wrong datatype - with self.assertRaisesOpError( - "TensorArray dtype is float but Op is trying to write dtype string"): - ta.write(-1, "wrong_type_scalar").flow.eval() - - # Test writing to a negative index - with self.assertRaisesOpError( - "Tried to write to index -1 but array is not " - "resizeable and size is: 3"): - ta.write(-1, 3.0).flow.eval() + if in_graph_mode: + with self.assertRaisesOpError( + "TensorArray dtype is float but Op is trying to write " + "dtype string"): + self.evaluate(ta.write(0, "wrong_type_scalar").flow) + else: + with self.assertRaisesOpError( + "TensorArray dtype is float32 but Op is trying to write " + "dtype string"): + self.evaluate(ta.write(0, "wrong_type_scalar").flow) + + if context.in_graph_mode(): + with self.assertRaisesOpError( + "Tried to write to index -1 but array is not " + "resizeable and size is: 3"): + self.evaluate(ta.write(-1, 3.0).flow) + else: + with self.assertRaisesOpError( + r"Writing to negative indices \(index -1\) is not allowed."): + self.evaluate(ta.write(-1, 3.0).flow) # Test reading from too large an index with self.assertRaisesOpError( "Tried to write to index 3 but array is not " "resizeable and size is: 3"): - ta.write(3, 3.0).flow.eval() + self.evaluate(ta.write(3, 3.0).flow) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayReadWrongIndexOrDataTypeFails(self): with self.test_session(use_gpu=True): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, tensor_array_name="foo", size=3) + ta = _make_ta(3, "foo", dtype=dtypes.float32) w0 = ta.write(0, [[4.0, 5.0]]) - # Test reading wrong datatype - r0_bad = gen_data_flow_ops._tensor_array_read_v3( - handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) - with self.assertRaisesOpError( - "TensorArray dtype is float but Op requested dtype double."): - r0_bad.eval() + # Test reading wrong datatype, which is only possible in graph mode + if context.in_graph_mode(): + r0_bad = gen_data_flow_ops._tensor_array_read_v3( + handle=w0.handle, index=0, dtype=dtypes.float64, flow_in=w0.flow) + with self.assertRaisesOpError( + "TensorArray dtype is float but Op requested dtype double."): + r0_bad.eval() # Test reading from a different index than the one we wrote to - r1 = w0.read(1) with self.assertRaisesOpError( "Could not read from TensorArray index 1 because " "it has not yet been written to."): - r1.eval() + self.evaluate(w0.read(1)) - # Test reading from a negative index - with self.assertRaisesOpError( - r"Tried to read from index -1 but array size is: 3"): - ta.read(-1).eval() + # Test reading from a negative index, which is not allowed + if context.in_graph_mode(): + with self.assertRaisesOpError( + r"Tried to read from index -1 but array size is: 3"): + self.evaluate(ta.read(-1)) + else: + with self.assertRaisesOpError( + r"Reading from negative indices \(index -1\) is not allowed."): + self.evaluate(ta.read(-1)) # Test reading from too large an index with self.assertRaisesOpError( "Tried to read from index 3 but array size is: 3"): - ta.read(3).eval() + self.evaluate(ta.read(3)) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteMultipleFails(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -428,8 +453,12 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError( "Could not write to TensorArray index 2 because " "it has already been written to."): - ta.write(2, 3.0).write(2, 3.0).flow.eval() + if context.in_graph_mode(): + self.evaluate(ta.write(2, 3.0).write(2, 3.0).flow) + else: + self.evaluate(ta.write(2, 3.0).write(2, 3.0)) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayConcatIncompatibleShapesFails(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -444,7 +473,7 @@ class TensorArrayTest(test.TestCase): with self.assertRaisesOpError( "Concat saw a scalar shape at index 0 but requires at least vectors"): - w3.concat().eval() + self.evaluate(w3.concat()) ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -456,45 +485,58 @@ class TensorArrayTest(test.TestCase): w2 = w1.write(1, [4.0]) w3 = w2.write(2, [[3.0]]) - with self.assertRaisesOpError( - r"TensorArray has inconsistent shapes. Index 0 has " - r"\(excepting dimension 0\) shape: \[\] but index 2 has \(excepting " - r"dimension 0\) shape: \[1\]"): - w3.concat().eval() + # The eager-mode implementation just passes up array_op.concat's error + # message. + if context.in_graph_mode(): + with self.assertRaisesOpError( + r"TensorArray has inconsistent shapes. Index 0 has " + r"\(excepting dimension 0\) shape: \[\] but index 2 has " + r"\(excepting dimension 0\) shape: \[1\]"): + self.evaluate(w3.concat()) + else: + with self.assertRaisesOpError( + r".*Ranks of all input tensors should match: shape\[0\] " + r"= \[1\] vs\. shape\[2\] = \[1,1\].*"): + self.evaluate(w3.concat()) + @test_util.run_in_graph_and_eager_modes() def testTensorArraySplitIncompatibleShapesFails(self): with self.test_session(use_gpu=True): - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, - tensor_array_name="foo", - size=3, - infer_shape=False) - + in_graph_mode = context.in_graph_mode() + ta = _make_ta(3, "foo") with self.assertRaisesOpError( r"Expected lengths to be a vector, received shape: \[\]"): - lengths = array_ops.placeholder(dtypes.int64) - ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1}) + if in_graph_mode: + lengths = array_ops.placeholder(dtypes.int64) + ta.split([1.0, 2.0, 3.0], lengths).flow.eval(feed_dict={lengths: 1}) + else: + self.evaluate(ta.split([1.0, 2.0, 3.0], 1)) with self.assertRaisesOpError( r"Expected sum of lengths to be equal to values.shape\[0\], " r"but sum of lengths is 1 and value's shape is: \[3\]"): - ta.split([1.0, 2.0, 3.0], [1]).flow.eval() + if in_graph_mode: + self.evaluate(ta.split([1.0, 2.0, 3.0], [1]).flow) + else: + self.evaluate(ta.split([1.0, 2.0, 3.0], [1])) + ta = _make_ta(1, "baz") with self.assertRaisesOpError( r"Expected value to be at least a vector, but received shape: \[\]"): - ta.split(1.0, [1]).flow.eval() - - ta = tensor_array_ops.TensorArray( - dtype=dtypes.float32, - tensor_array_name="foo", - size=2, - infer_shape=False) + if in_graph_mode: + self.evaluate(ta.split(1.0, [1]).flow) + else: + self.evaluate(ta.split(1.0, [1])) + ta = _make_ta(2, "buz") with self.assertRaisesOpError( r"TensorArray's size is not equal to the size of lengths " r"\(2 vs. 1\), and the TensorArray is not marked as " r"dynamically resizeable"): - ta.split([1.0], [1]).flow.eval() + if in_graph_mode: + self.evaluate(ta.split([1.0], [1]).flow) + else: + self.evaluate(ta.split([1.0], [1])) def _testTensorArrayWriteGradientAddMultipleAdds(self, dtype): with self.test_session(use_gpu=True): @@ -535,6 +577,7 @@ class TensorArrayTest(test.TestCase): dtypes.complex64, dtypes.complex128): self._testTensorArrayWriteGradientAddMultipleAdds(dtype) + @test_util.run_in_graph_and_eager_modes() def testMultiTensorArray(self): with self.test_session(use_gpu=True): h1 = tensor_array_ops.TensorArray( @@ -548,7 +591,8 @@ class TensorArrayTest(test.TestCase): w2 = h2.write(0, 5.0) r2 = w2.read(0) r = r1 + r2 - self.assertAllClose(9.0, r.eval()) + val = self.evaluate(r) + self.assertAllClose(9.0, val) def _testTensorArrayGradientWriteReadType(self, dtype): with self.test_session(use_gpu=True) as session: @@ -637,6 +681,7 @@ class TensorArrayTest(test.TestCase): def testTensorArrayGradientWritePackConcatAndRead(self): self._testTensorArrayGradientWritePackConcatAndRead() + @test_util.run_in_graph_and_eager_modes() def testTensorArrayReadTwice(self): with self.test_session(use_gpu=True): value = constant_op.constant([[1.0, -1.0], [10.0, -10.0]]) @@ -646,13 +691,12 @@ class TensorArrayTest(test.TestCase): w_readonce = ta_readonce.unstack(value) r0_readonce = w_readonce.read(0) - with ops.control_dependencies([r0_readonce]): - r1_readonce = w_readonce.read(0) with self.assertRaisesOpError( r"Could not read index 0 twice because it was cleared after a " r"previous read \(perhaps try setting clear_after_read = false\?\)"): - r1_readonce.eval() + with ops.control_dependencies([r0_readonce]): + self.evaluate(w_readonce.read(0)) ta_readtwice = tensor_array_ops.TensorArray( dtype=dtypes.float32, @@ -664,7 +708,7 @@ class TensorArrayTest(test.TestCase): with ops.control_dependencies([r0_readtwice]): r1_readtwice = w_readtwice.read(0) - self.assertAllEqual([1.0, -1.0], r1_readtwice.eval()) + self.assertAllEqual([1.0, -1.0], self.evaluate(r1_readtwice)) def _testTensorArrayGradientUnpackRead(self): with self.test_session(use_gpu=True) as session: @@ -741,20 +785,22 @@ class TensorArrayTest(test.TestCase): def testTensorArrayGradientDynamicUnpackRead(self): self._testTensorArrayGradientDynamicUnpackRead() + @test_util.run_in_graph_and_eager_modes() def testCloseTensorArray(self): - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) - c1 = ta.close() - session.run(c1) + self.evaluate(ta.close()) + @test_util.run_in_graph_and_eager_modes() def testSizeTensorArray(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( dtype=dtypes.float32, tensor_array_name="foo", size=3) s = ta.size() - self.assertAllEqual(3, s.eval()) + self.assertAllEqual(3, self.evaluate(s)) + @test_util.run_in_graph_and_eager_modes() def testWriteCloseTensorArray(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -764,48 +810,62 @@ class TensorArrayTest(test.TestCase): infer_shape=False) w0 = ta.write(0, [[4.0, 5.0]]) w1 = w0.write(1, [3.0]) - w1.close().run() # Expected to run without problems + self.evaluate(w1.close()) # Expected to run without problems def _testWhileLoopWritePackGradients(self, dynamic_size, dtype): np_dtype = dtype.as_numpy_dtype - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): + def func(v0, state0, var): + ta = tensor_array_ops.TensorArray( + dtype=dtype, + tensor_array_name="foo", + size=0 if dynamic_size else 3, + dynamic_size=dynamic_size) + time_0 = array_ops.identity(0) + + def body(time, ta_t, state): + sliced = array_ops.slice( + v0, begin=array_ops.stack([time, 0]), size=[1, -1]) + sliced = array_ops.squeeze(sliced) + out = sliced + var + state + state += sliced + ta_t = ta_t.write(time, out) + return (time + 1, ta_t, state) + + (unused_0, h_final, unused_2) = control_flow_ops.while_loop( + cond=lambda time, unused_1, unused_2: time < 3, + body=body, + loop_vars=(time_0, ta, state0), + shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(), + tensor_shape.unknown_shape()), + parallel_iterations=3) + vout = h_final.stack() + return vout + v0 = array_ops.identity(np.arange(3 * 5, dtype=np_dtype).reshape(3, 5)) - var = variables.Variable(np.arange(100, 105, dtype=np_dtype)) state0 = array_ops.identity(np.array([1] * 5, dtype=np_dtype)) - ta = tensor_array_ops.TensorArray( - dtype=dtype, - tensor_array_name="foo", - size=0 if dynamic_size else 3, - dynamic_size=dynamic_size) - time_0 = array_ops.identity(0) - - def body(time, ta_t, state): - sliced = array_ops.slice( - v0, begin=array_ops.stack([time, 0]), size=[1, -1]) - sliced = array_ops.squeeze(sliced) - out = sliced + var + state - state += sliced - ta_t = ta_t.write(time, out) - return (time + 1, ta_t, state) - - (unused_0, h_final, unused_2) = control_flow_ops.while_loop( - cond=lambda time, unused_1, unused_2: time < 3, - body=body, - loop_vars=(time_0, ta, state0), - shape_invariants=(time_0.get_shape(), tensor_shape.unknown_shape(), - tensor_shape.unknown_shape()), - parallel_iterations=3) - vout = h_final.stack() - + init_val = np.arange(100, 105, dtype=np_dtype) + var = variable_scope.get_variable( + "var", + shape=init_val.shape, + dtype=np_dtype, + initializer=init_ops.constant_initializer(init_val)) + + vout = func(v0, state0, var) grad_val = -np.arange(3 * 5, dtype=np_dtype).reshape(3, 5) - v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0] - state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0] - var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0] + if context.in_graph_mode(): + v0_grad = gradients_impl.gradients([vout], [v0], [grad_val])[0] + state0_grad = gradients_impl.gradients([vout], [state0], [grad_val])[0] + var_grad = gradients_impl.gradients([vout], [var], [grad_val])[0] + variables.global_variables_initializer().run() + else: + grad_fn = backprop.gradients_function(func) + v0_grad, state0_grad, var_grad = grad_fn(v0, state0, var, dy=grad_val) - variables.global_variables_initializer().run() state0_t, var_t, v0_t, vout_t, v0_grad_t, var_grad_t, state0_grad_t = ( - session.run([state0, var, v0, vout, v0_grad, var_grad, state0_grad])) - just_v0_grad_t, = session.run([v0_grad]) + self.evaluate( + ([state0, var, v0, vout, v0_grad, var_grad, state0_grad]))) + just_v0_grad_t = self.evaluate(v0_grad) # state = [ state0 | state0 + v0[0] | state0 + v0[0] + v0[1] ] # vout = [ v0[0] + var + state[0] | @@ -838,6 +898,7 @@ class TensorArrayTest(test.TestCase): self.assertAllClose(grad_val.sum(axis=0), var_grad_t) self.assertAllClose(grad_val.sum(axis=0), state0_grad_t) + @test_util.run_in_graph_and_eager_modes() def testWhileLoopWritePackGradients(self): self._testWhileLoopWritePackGradients( dynamic_size=False, dtype=dtypes.float32) @@ -849,38 +910,45 @@ class TensorArrayTest(test.TestCase): self._testWhileLoopWritePackGradients( dynamic_size=True, dtype=dtypes.float32) + @test_util.run_in_graph_and_eager_modes() def testGradSerialTwoLoops(self): with self.test_session(use_gpu=True): - num_steps = 100 - acc = tensor_array_ops.TensorArray( - dtype=dtypes.float32, - size=num_steps, - clear_after_read=False, - element_shape=tensor_shape.scalar()) - i = constant_op.constant(0, name="i") - x = constant_op.constant(2.0, name="x") + def loop(x): + num_steps = 100 + acc = tensor_array_ops.TensorArray( + dtype=dtypes.float32, + size=num_steps, + clear_after_read=False, + element_shape=tensor_shape.scalar()) + i = constant_op.constant(0, name="i") + + c = lambda i, acc: i < 5 - c = lambda i, acc: i < 5 + def b(i, acc): + x1 = control_flow_ops.cond( + math_ops.equal(i, 0), lambda: x, + lambda: math_ops.multiply(acc.read(i - 1), 2.0)) + return i + 1, acc.write(i, x1) - def b(i, acc): - x1 = control_flow_ops.cond( - math_ops.equal(i, 0), lambda: x, - lambda: math_ops.multiply(acc.read(i - 1), 2.0)) - return i + 1, acc.write(i, x1) + i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc]) - i1, acc1 = control_flow_ops.while_loop(c, b, [i, acc]) + z = constant_op.constant(0.0) - z = constant_op.constant(0.0) + def fn(i, acc): + return i + 1, acc.write(i, z) - def fn(i, acc): - return i + 1, acc.write(i, z) + _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn, + [i1, acc1]) - _, acc2 = control_flow_ops.while_loop(lambda i, acc: i < num_steps, fn, - [i1, acc1]) + r = acc2.stack() + return r - r = acc2.stack() - grad = gradients_impl.gradients(r, [x])[0] - self.assertAllClose(31.0, grad.eval()) + x = constant_op.constant(2.0, name="x") + if context.in_graph_mode(): + grad = gradients_impl.gradients(loop(x), [x])[0] + else: + grad = backprop.gradients_function(loop)(x)[0] + self.assertAllClose(31.0, self.evaluate(grad)) def testSumOfTwoReadVariablesWithoutRepeatGrad(self): with self.test_session(use_gpu=True) as session: @@ -1019,6 +1087,7 @@ class TensorArrayTest(test.TestCase): r5 = w5.read(0) self.assertAllEqual([5, 4, 2, 3], r5.get_shape().as_list()) + @test_util.run_in_graph_and_eager_modes() def _testUnpackShape(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1035,8 +1104,15 @@ class TensorArrayTest(test.TestCase): c1 = constant_op.constant([4.0, 5.0]) w1 = w0.write(3, c1) - r1 = w1.read(0) - self.assertAllEqual(c1.get_shape(), r1.get_shape()) + + with self.assertRaisesOpError( + r"Could not read index 0 twice because it was cleared after a " + r"previous read \(perhaps try setting clear_after_read = false\?\)"): + with ops.control_dependencies([r0]): + self.evaluate(w1.read(0)) + + r1 = w1.read(1) + self.assertAllEqual(c1.get_shape(), r1.shape) c2 = constant_op.constant([4.0, 5.0, 6.0]) with self.assertRaises(ValueError): @@ -1045,6 +1121,7 @@ class TensorArrayTest(test.TestCase): def testUnpackShape(self): self._testUnpackShape() + @test_util.run_in_graph_and_eager_modes() def testSplitShape(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1066,10 +1143,14 @@ class TensorArrayTest(test.TestCase): infer_shape=True) w0 = ta1.split(value, [1, 2]) r0 = w0.read(0) - self.assertEqual(r0.get_shape().ndims, None) - self.assertEqual( - tensor_shape.TensorShape( - ta1.handle.op.get_attr("element_shape")).ndims, None) + if context.in_graph_mode(): + self.assertEqual(r0.get_shape().ndims, None) + self.assertEqual( + tensor_shape.TensorShape( + ta1.handle.op.get_attr("element_shape")).ndims, None) + else: + self.assertEqual((1, 2), r0.get_shape()) + self.assertEqual((2, 2), w0.read(1).get_shape()) def testWriteUnknownShape(self): with self.test_session(use_gpu=True): @@ -1137,6 +1218,8 @@ class TensorArrayTest(test.TestCase): def testTensorArrayEvalEmpty(self): self._testTensorArrayEvalEmpty() + # this test is ill-defined for Eager mode --- unpacking an empty tensor + # gives an empty list / there is not equivalent of "mark_used" in Eager def _testTensorArrayEvalEmptyWithDefault(self): with self.test_session(use_gpu=True): ta = tensor_array_ops.TensorArray( @@ -1180,6 +1263,7 @@ class TensorArrayTest(test.TestCase): self.assertAllEqual([10.0, -10.0], read_vals[1]) self.assertAllEqual([[2.0, 3.0], [4.0, 5.0]], grad_vals[0]) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayWriteGatherAndGradients(self): with self.test_session(use_gpu=True) as session: ta = tensor_array_ops.TensorArray( @@ -1188,16 +1272,23 @@ class TensorArrayTest(test.TestCase): size=0, dynamic_size=True) - values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)]) - indices = constant_op.constant([1, 8]) - - w = ta.unstack(values) - g = w.gather(indices) + def func(values): + indices = constant_op.constant([1, 8]) + w = ta.unstack(values) + g = w.gather(indices) + return g + values = constant_op.constant([[1.0 * x, -1.0 * x] for x in range(10)]) + g = func(values) + grad_ys = [[[2.0, 3.0], [4.0, 5.0]]] # Test combined gradients + aggregation of read(0) - grad = gradients_impl.gradients( - ys=[g], xs=[values], grad_ys=[[[2.0, 3.0], [4.0, 5.0]]]) - g_vals, grad_vals = session.run([[g], grad]) + if context.in_graph_mode(): + grad = gradients_impl.gradients(ys=[g], xs=[values], grad_ys=grad_ys) + g_vals, grad_vals = session.run([[g], grad]) + else: + g_vals = [g] + grad_vals = backprop.gradients_function(func)( + values, dy=constant_op.constant(grad_ys[0], dtype=dtypes.float32)) # Gradients for 8 of the 10 unread components are zero. expected_grad = np.zeros((10, 2)) @@ -1316,8 +1407,9 @@ class TensorArrayTest(test.TestCase): self.assertFalse( [s for s in dev_stats[d] if "/TensorArray" in s.node_name]) + @test_util.run_in_graph_and_eager_modes() def testTensorArrayIdentity(self): - with self.test_session(use_gpu=True) as session: + with self.test_session(use_gpu=True): ta0 = tensor_array_ops.TensorArray(dtype=dtypes.float32, size=2, infer_shape=False) ta1 = tensor_array_ops.TensorArray(dtype=dtypes.int32, size=4, @@ -1326,8 +1418,10 @@ class TensorArrayTest(test.TestCase): ta0 = ta0.write(0, 0.) ta1 = ta1.write(0, 1) - v0 = variables.Variable(0) - v1 = variables.Variable(0) + v0 = variable_scope.get_variable( + "v0", shape=(), initializer=init_ops.zeros_initializer()) + v1 = variable_scope.get_variable( + "v1", shape=(), initializer=init_ops.zeros_initializer()) with ops.control_dependencies([v0.assign_add(1)]): ta0 = ta0.identity() @@ -1344,17 +1438,21 @@ class TensorArrayTest(test.TestCase): # Tests correct properties on new TensorArrays. self.assertEqual(dtypes.float32, ta0.dtype) self.assertEqual(dtypes.int32, ta1.dtype) - self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape()) + if context.in_graph_mode(): + self.assertEqual(tensor_shape.unknown_shape(), read0.get_shape()) + else: + self.assertEqual(tensor_shape.scalar(), read1.get_shape()) self.assertEqual(tensor_shape.scalar(), read1.get_shape()) - variables.global_variables_initializer().run() + if context.in_graph_mode(): + variables.global_variables_initializer().run() - read0_v, read1_v, size0_v, size1_v = session.run( - (read0, read1, size0, size1)) + read0_v, read1_v, size0_v, size1_v = self.evaluate((read0, read1, size0, + size1)) # Tests that the control dependencies was added and executed. - self.assertEqual(1, v0.eval()) - self.assertEqual(1, v1.eval()) + self.assertEqual(1, self.evaluate(v0)) + self.assertEqual(1, self.evaluate(v1)) # Tests correct TensorArray. self.assertEqual(read0_v, 0) diff --git a/tensorflow/python/kernel_tests/topk_op_test.py b/tensorflow/python/kernel_tests/topk_op_test.py index a8e7799cabd6ce5fe7366bd77c560cbfc840d95d..efb5b9f3641ceaebf1fd5285486b4a9bb93615cf 100644 --- a/tensorflow/python/kernel_tests/topk_op_test.py +++ b/tensorflow/python/kernel_tests/topk_op_test.py @@ -100,6 +100,13 @@ class TopKTest(test.TestCase): inputs = [[0.1, 0.3, 0.2, 0.4], [0.1, 0.3, 0.4, 0.2]] self._validateTopK(inputs, 2, [[0.4, 0.3], [0.4, 0.3]], [[3, 1], [2, 1]]) + def testTop3(self): + k = 5 + inputs = np.random.permutation(np.linspace(0, 100, 6140, dtype=np.float64)) + indices = np.argsort(-inputs)[:k] + values = -np.sort(-inputs)[:k] + self._validateTopK(inputs, k, values, indices) + def _testLargeSort(self, dtype): b = 10 n = 5000 diff --git a/tensorflow/python/kernel_tests/transpose_op_test.py b/tensorflow/python/kernel_tests/transpose_op_test.py index 3b352937c82138d89c63a11b3b237719831ee7e7..c551d9c3d056b50600d1331749ba865439748f7e 100644 --- a/tensorflow/python/kernel_tests/transpose_op_test.py +++ b/tensorflow/python/kernel_tests/transpose_op_test.py @@ -317,6 +317,19 @@ class TransposeTest(test.TestCase): np.arange(0, 8).reshape([2, 4]).astype(np.float32), np.array([1, 0]).astype(np.int32)) + def testPermType(self): + for perm_dtype in [np.int64, np.int32]: + x = np.arange(0, 8).reshape([2, 4]).astype(np.float32) + p = np.array([1, 0]).astype(perm_dtype) + np_ans = np.copy(x).transpose(p) + with self.test_session(use_gpu=True): + inx = ops.convert_to_tensor(x) + inp = constant_op.constant(p) + y = array_ops.transpose(inx, inp) + tf_ans = y.eval() + self.assertShapeEqual(np_ans, y) + self.assertAllEqual(np_ans, tf_ans) + def testHalf(self): self._compare(np.arange(0, 21).reshape([3, 7]).astype(np.float16)) self._compare(np.arange(0, 210).reshape([2, 3, 5, 7]).astype(np.float16)) diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py index 0ea58b44026718ea1bdc23ac791593be98846fe8..bd4b12b7e8aee91eeabc677d9e1bfd33cde7911d 100644 --- a/tensorflow/python/kernel_tests/variable_scope_test.py +++ b/tensorflow/python/kernel_tests/variable_scope_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import gc + import numpy from tensorflow.python.eager import context @@ -39,7 +41,12 @@ from tensorflow.python.platform import test class VariableScopeTest(test.TestCase): - @test_util.run_in_graph_and_eager_modes() + def tearDown(self): + gc.collect() + # This will only contain uncollectable garbage, i.e. reference cycles + # involving objects with __del__ defined. + self.assertEqual(0, len(gc.garbage)) + def testGetVar(self): vs = variable_scope._get_default_variable_store() v = vs.get_variable("v", [1]) @@ -52,7 +59,6 @@ class VariableScopeTest(test.TestCase): v1 = vs.get_variable("v", [1], use_resource=True) self.assertTrue(isinstance(v1, resource_variable_ops.ResourceVariable)) - @test_util.run_in_graph_and_eager_modes() def testNameExists(self): vs = variable_scope._get_default_variable_store() # No check by default, so we can both create and get existing names. @@ -60,17 +66,15 @@ class VariableScopeTest(test.TestCase): v1 = vs.get_variable("v", [1]) self.assertEqual(v, v1) - if context.in_graph_mode(): - # When reuse is False, we fail when variables are already there. - vs.get_variable("w", [1], reuse=False) # That's ok. - with self.assertRaises(ValueError): - vs.get_variable("v", [1], reuse=False) # That fails. - # When reuse is True, we fail when variables are new. - vs.get_variable("v", [1], reuse=True) # That's ok. - with self.assertRaises(ValueError): - vs.get_variable("u", [1], reuse=True) # That fails. + # When reuse is False, we fail when variables are already there. + vs.get_variable("w", [1], reuse=False) # That's ok. + with self.assertRaises(ValueError): + vs.get_variable("v", [1], reuse=False) # That fails. + # When reuse is True, we fail when variables are new. + vs.get_variable("v", [1], reuse=True) # That's ok. + with self.assertRaises(ValueError): + vs.get_variable("u", [1], reuse=True) # That fails. - @test_util.run_in_graph_and_eager_modes() def testNamelessStore(self): vs = variable_scope._get_default_variable_store() vs.get_variable("v1", [2]) @@ -224,10 +228,12 @@ class VariableScopeTest(test.TestCase): self.assertAllClose(self.evaluate(losses[1]), 0.4) self.assertAllClose(self.evaluate(losses[2]), 0.5) with variable_scope.variable_scope("foo", reuse=True): - v = variable_scope.get_variable("v", - []) # "v" is alredy there, reused - losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES) - self.assertEqual(3, len(losses)) # No new loss added. + # reuse=True is for now only supported when eager execution is disabled. + if context.in_graph_mode(): + v = variable_scope.get_variable("v", + []) # "v" is alredy there, reused + losses = ops.get_collection(ops.GraphKeys.REGULARIZATION_LOSSES) + self.assertEqual(3, len(losses)) # No new loss added. @test_util.run_in_graph_and_eager_modes() def testInitializeFromValue(self): @@ -439,20 +445,20 @@ class VariableScopeTest(test.TestCase): with variable_scope.variable_scope(vs, reuse=False) as jump_no_reuse: self.assertFalse(jump_no_reuse.reuse) - @test_util.run_in_graph_and_eager_modes() def testVarScopeGetOrCreateReuse(self): - def test_value(value): - x = constant_op.constant(value) - with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar", - reuse=variable_scope.AUTO_REUSE): - _ = state_ops.assign(variable_scope.get_variable("var", []), x) - with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar", - reuse=variable_scope.AUTO_REUSE): - _ = variable_scope.get_variable("var", []) - self.assertEqual(value, self.evaluate(x)) - test_value(42.) # Variable is created. - test_value(13.) # Variable is reused hereafter. - test_value(17.) + with self.test_session(): + def test_value(value): + x = constant_op.constant(value) + with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar", + reuse=variable_scope.AUTO_REUSE): + _ = state_ops.assign(variable_scope.get_variable("var", []), x) + with variable_scope.variable_scope("testVarScopeGetOrCreateReuse_bar", + reuse=variable_scope.AUTO_REUSE): + _ = variable_scope.get_variable("var", []) + self.assertEqual(value, x.eval()) + test_value(42.) # Variable is created. + test_value(13.) # Variable is reused hereafter. + test_value(17.) def testVarOpScope(self): with self.test_session(): @@ -745,9 +751,10 @@ class VariableScopeTest(test.TestCase): ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) # Check that local variable respects `reuse`. - with variable_scope.variable_scope(outer, "default", reuse=True): - self.assertEqual( - variable_scope.get_local_variable("w", []).name, "outer/w:0") + if context.in_graph_mode(): + with variable_scope.variable_scope(outer, "default", reuse=True): + self.assertEqual( + variable_scope.get_local_variable("w", []).name, "outer/w:0") def testGetVarWithDevice(self): g = ops.Graph() diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py index 99a30657ef73763b43da1361a38283e318fb5f87..8c2ee1f103b66df3ad26897f59b2b51f8b0a6500 100644 --- a/tensorflow/python/layers/base.py +++ b/tensorflow/python/layers/base.py @@ -122,16 +122,7 @@ class Layer(object): self._inbound_nodes = [] self._outbound_nodes = [] - # Determine layer name (non-unique). - if isinstance(name, vs.VariableScope): - base_name = name.name - else: - base_name = name - self._name = name - if not name: - base_name = _to_snake_case(self.__class__.__name__) - self._name = _unique_layer_name(base_name) - self._base_name = base_name + self._init_set_name(name) # Determine variable scope. scope = kwargs.get('_scope') @@ -147,6 +138,17 @@ class Layer(object): batch_size = kwargs.get('batch_size') self._batch_input_shape = (batch_size,) + tuple(kwargs['input_shape']) + def _init_set_name(self, name): + # Determine layer name (non-unique). + if isinstance(name, vs.VariableScope): + base_name = name.name + else: + base_name = name + self._name = name + if not name: + self._name, base_name = self._make_unique_name() + self._base_name = base_name + @property def dtype(self): return self._dtype @@ -399,6 +401,12 @@ class Layer(object): """ return input_shape + def _make_unique_name(self, name_uid_map=None, avoid_names=None): + base_name = _to_snake_case(self.__class__.__name__) + name = _unique_layer_name(base_name, name_uid_map=name_uid_map, + avoid_names=avoid_names) + return (name, base_name) + def _set_scope(self, scope=None): if self._scope is None: # If constructed with _scope=None, lazy setting of scope. @@ -413,7 +421,8 @@ class Layer(object): def add_variable(self, name, shape, dtype=None, initializer=None, regularizer=None, - trainable=True, constraint=None): + trainable=True, constraint=None, + partitioner=None): """Adds a new variable to the layer, or gets an existing one; returns it. Arguments: @@ -426,9 +435,19 @@ class Layer(object): "trainable_variables" (e.g. variables, biases) or "non_trainable_variables" (e.g. BatchNorm mean, stddev). constraint: constraint instance (callable). + partitioner: (optional) partitioner instance (callable). If + provided, when the requested variable is created it will be split + into multiple partitions according to `partitioner`. In this case, + an instance of `PartitionedVariable` is returned. Available + partitioners include `tf.fixed_size_partitioner` and + `tf.variable_axis_size_partitioner`. For more details, see the + documentation of `tf.get_variable` and the "Variable Partitioners + and Sharding" section of the API guide. Returns: - The created variable. + The created variable. Usually either a `Variable` or `ResourceVariable` + instance. If `partitioner` is not `None`, a `PartitionedVariable` + instance is returned. Raises: RuntimeError: If called in Eager mode with regularizers. @@ -455,7 +474,8 @@ class Layer(object): initializer=initializer, dtype=dtypes.as_dtype(dtype), constraint=constraint, - trainable=trainable and self.trainable) + trainable=trainable and self.trainable, + partitioner=partitioner) if variable in existing_variables: return variable if regularizer: @@ -508,6 +528,7 @@ class Layer(object): input_list = nest.flatten(inputs) in_graph_mode = context.in_graph_mode() + in_deferred_mode = isinstance(input_list[0], _DeferredTensor) # Ensure the Layer, if being reused, is working with inputs from # the same graph as where it was created. if in_graph_mode: @@ -515,6 +536,7 @@ class Layer(object): ops._get_graph_from_inputs(input_list, graph=self.graph) # pylint: disable=protected-access except ValueError as e: raise ValueError('Input graph and Layer graph are not the same: %s' % e) + if in_graph_mode or in_deferred_mode: user_kwargs = copy.copy(kwargs) # Handle Keras mask propagation from previous layer to current layer. @@ -553,6 +575,7 @@ class Layer(object): raise ValueError('activity_regularizer currently unsupported in ' 'Eager mode. Found an activity_regularizer in ' '%s(%s).' % (self.__class__.__name__, self)) + if not in_graph_mode and not in_deferred_mode: # TODO(agarwal): support _keras_history in Eager mode. for x in input_list: if hasattr(x, '_keras_history'): @@ -581,13 +604,26 @@ class Layer(object): if call_has_scope_arg: kwargs['scope'] = scope # Check input assumptions set after layer building, e.g. input shape. - if in_graph_mode: + if in_graph_mode or in_deferred_mode: self._assert_input_compatibility(inputs) - outputs = self.call(inputs, *args, **kwargs) - if outputs is None: - raise ValueError('A layer\'s `call` method should return a Tensor ' - 'or a list of Tensors, not None.') + if not in_deferred_mode: + outputs = self.call(inputs, *args, **kwargs) + if outputs is None: + raise ValueError('A layer\'s `call` method should return a Tensor ' + 'or a list of Tensors, not None.') + else: + # Deferred mode behavior: use `_compute_output_shape` to + # infer the number of outputs of the layer and their shapes. + output_shapes = self._compute_output_shape(input_shapes) + output_shapes = nest.flatten(output_shapes) + outputs = [ + # TODO(fchollet): name the deferred tensors? + _DeferredTensor(shape=shape, dtype=self._dtype) + for shape in output_shapes + ] + if len(outputs) == 1: + outputs = outputs[0] if in_graph_mode: # Apply activity regularization. @@ -600,16 +636,18 @@ class Layer(object): activity_regularization = self._activity_regularizer(output) self.add_loss(activity_regularization) - # Handle mask computation and propagation to the next layer. - if hasattr(self, 'compute_mask'): - output_mask = self.compute_mask(inputs, previous_mask) - if isinstance(outputs, list): - if output_mask is None: - output_mask = [None for _ in range(len(outputs))] - for x, m in zip(outputs, output_mask): - x._keras_mask = m # pylint: disable=protected-access - else: - outputs._keras_mask = output_mask # pylint: disable=protected-access + if not in_deferred_mode: + # TODO(fchollet): consider how masking will work with deferred mode. + # Handle mask computation and propagation to the next layer. + if hasattr(self, 'compute_mask'): + output_mask = self.compute_mask(inputs, previous_mask) + if isinstance(outputs, list): + if output_mask is None: + output_mask = [None for _ in range(len(outputs))] + for x, m in zip(outputs, output_mask): + x._keras_mask = m # pylint: disable=protected-access + else: + outputs._keras_mask = output_mask # pylint: disable=protected-access if in_graph_mode: # If all input tensors have history metadata, @@ -631,14 +669,16 @@ class Layer(object): else: outputs = output_ls_copy + # Update global default collections. + _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) + + if in_deferred_mode or in_graph_mode: + if _have_all_keras_metadata(inputs): # Add an inbound node to the layer, so it can keep track of this call. # This updates the layer history of the output tensor(s). self._add_inbound_node( input_tensors=inputs, output_tensors=outputs, arguments=user_kwargs) - # Update global default collections. - _add_elements_to_collection(self.updates, ops.GraphKeys.UPDATE_OPS) - self.built = True return outputs @@ -692,7 +732,6 @@ class Layer(object): arguments: dictionary of keyword arguments that were passed to the `call` method of the layer at the call that created the node. """ - assert context.in_graph_mode() input_tensors = nest.flatten(input_tensors) output_tensors = nest.flatten(output_tensors) @@ -1251,6 +1290,34 @@ class Node(object): } +class _DeferredTensor(object): + """Tensor-like object used to build graphs of layers in Eager mode. + + When calling a layer on a DeferredTensor, the layer will not perform any + computation and will simply perfom shape inference to return new + DeferredTensors with appropriate shape information. Thus DeferredTensor + behaves like a graph-mode Tensor when manipulated by layers. + """ + + def __init__(self, shape, dtype, name=None): + self.shape = tensor_shape.TensorShape(shape) + self.dtype = dtypes.as_dtype(dtype) + self.name = name + + def get_shape(self): + return self.shape + + def __str__(self): + return "DeferredTensor('%s', shape=%s, dtype=%s)" % (self.name, + self.get_shape(), + self.dtype.name) + + def __repr__(self): + return "<_DeferredTensor '%s' shape=%s dtype=%s>" % (self.name, + self.get_shape(), + self.dtype.name) + + class InputLayer(Layer): """Layer to be used as an entry point into a Network (a graph of layers). @@ -1283,8 +1350,6 @@ class InputLayer(Layer): input_tensor=None, sparse=False, name=None): - if context.in_eager_mode(): - raise RuntimeError('InputLayer not supported in Eager mode.') super(InputLayer, self).__init__(dtype=dtype, name=name) self.built = True self.sparse = sparse @@ -1299,16 +1364,24 @@ class InputLayer(Layer): else: batch_input_shape = None - if sparse: - input_tensor = array_ops.sparse_placeholder( + if context.in_eager_mode(): + # In eager mode, create a temporary placeholder to call the layer on. + input_tensor = _DeferredTensor( shape=batch_input_shape, dtype=dtype, name=self.name) else: - input_tensor = array_ops.placeholder( - shape=batch_input_shape, - dtype=dtype, - name=self.name) + # In graph mode, create a graph placeholder to call the layer on. + if sparse: + input_tensor = array_ops.sparse_placeholder( + shape=batch_input_shape, + dtype=dtype, + name=self.name) + else: + input_tensor = array_ops.placeholder( + shape=batch_input_shape, + dtype=dtype, + name=self.name) # For compatibility with Keras API. self.is_placeholder = True @@ -1375,8 +1448,6 @@ def Input( # pylint: disable=invalid-name Raises: RuntimeError: If called in Eager mode. """ - if context.in_eager_mode(): - raise RuntimeError('Input not supported in Eager mode.') input_layer = InputLayer( input_shape=shape, batch_size=batch_size, @@ -1440,22 +1511,15 @@ class Network(Layer): """ def __init__(self, inputs, outputs, name=None): # pylint: disable=super-init-not-called - # TODO(agarwal): Make Network work in Eager mode. if context.in_eager_mode(): - raise RuntimeError('Network not supported in Eager mode.') - # Set layer name and scope - if isinstance(name, vs.VariableScope): - base_name = name.name - else: - base_name = name - self._name = name - if not name: - base_name = _to_snake_case(self.__class__.__name__) - self._name = _unique_layer_name(base_name) + # TODO(fchollet): check that all inputs and outputs are DeferredTensors. + pass + + self._init_set_name(name) self._activity_regularizer = None - with vs.variable_scope(None, default_name=base_name) as captured_scope: + with vs.variable_scope( + None, default_name=self._base_name) as captured_scope: self._scope = captured_scope - self._base_name = base_name call_fn_args = estimator_util.fn_args(self.call) self._compute_previous_mask = ('mask' in call_fn_args or hasattr(self, 'compute_mask')) @@ -1919,16 +1983,17 @@ class Network(Layer): masks = [None for _ in range(len(inputs))] else: masks = nest.flatten(mask) - # Try to retrieve cached outputs if the layer has already been called - # on these exact inputs. - cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) - if cache_key in self._output_tensor_cache: - # Cache hit. - return self._output_tensor_cache[cache_key] - else: - # Cache miss: actually apply the network graph to the new inputs. - output_tensors, _, _ = self._run_internal_graph(inputs, masks) - return output_tensors + + if context.in_graph_mode(): + # Try to retrieve cached outputs if the layer has already been called + # on these exact inputs. + cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) + if cache_key in self._output_tensor_cache: + # Cache hit. + return self._output_tensor_cache[cache_key] + # Actually apply the network graph to the new inputs. + outputs, _ = self._run_internal_graph(inputs, masks) + return outputs def _compute_output_shape(self, input_shape): if isinstance(input_shape, list): @@ -2091,6 +2156,7 @@ class Network(Layer): if 'mask' in estimator_util.fn_args(layer.call): if 'mask' not in kwargs: kwargs['mask'] = computed_mask + output_tensors = nest.flatten( layer.call(computed_tensor, **kwargs)) if hasattr(layer, 'compute_mask'): @@ -2121,18 +2187,19 @@ class Network(Layer): ] layer.add_loss(regularization_losses, computed_tensors) - # Update model updates and losses: - # Keep track of updates that depend on the inputs - # (e.g. BN updates). - self.add_update(layer.get_updates_for(computed_tensors), inputs) - # Keep track of unconditional updates (e.g. a counter). - self.add_update(layer.get_updates_for(None), None) - # Keep track of losses that depend on the inputs - # (e.g. activity regularizers). - self.add_loss(layer.get_losses_for(computed_tensors), inputs) - # Keep track of unconditional losses - # (e.g. weight regularizers). - self.add_loss(layer.get_losses_for(None), None) + if context.in_graph_mode(): + # Update model updates and losses: + # Keep track of updates that depend on the inputs + # (e.g. BN updates). + self.add_update(layer.get_updates_for(computed_tensors), inputs) + # Keep track of unconditional updates (e.g. a counter). + self.add_update(layer.get_updates_for(None), None) + # Keep track of losses that depend on the inputs + # (e.g. activity regularizers). + self.add_loss(layer.get_losses_for(computed_tensors), inputs) + # Keep track of unconditional losses + # (e.g. weight regularizers). + self.add_loss(layer.get_losses_for(None), None) # Update tensor_map. for x, y, mask in zip(reference_output_tensors, output_tensors, @@ -2149,31 +2216,26 @@ class Network(Layer): output_tensors.append(tensor) output_masks.append(mask) - # Update cache; - # keys are based on ids on input tensors and inputs masks. - cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) - if len(output_tensors) == 1: output_tensors = output_tensors[0] - self._output_tensor_cache[cache_key] = output_tensors - else: - self._output_tensor_cache[cache_key] = output_tensors - - if len(output_masks) == 1: - output_masks = output_masks[0] - self._output_mask_cache[cache_key] = output_masks - else: - self._output_mask_cache[cache_key] = output_masks - - if output_shapes is not None: - input_shapes = [_static_shape(x) for x in inputs] - cache_key = _object_list_uid(input_shapes) - if len(output_shapes) == 1: + if output_shapes is not None: output_shapes = output_shapes[0] + if output_masks is not None: + output_masks = output_masks[0] + + if context.in_graph_mode(): + # Update cache; + # keys are based on ids on input tensors and inputs masks. + cache_key = _object_list_uid(inputs) + '_' + _object_list_uid(masks) + self._output_tensor_cache[cache_key] = output_tensors + if output_masks is not None: + self._output_mask_cache[cache_key] = output_masks + if output_shapes is not None: + input_shapes = [_static_shape(x) for x in inputs] + cache_key = _object_list_uid(input_shapes) self._output_shape_cache[cache_key] = output_shapes - else: - self._output_shape_cache[cache_key] = output_shapes - return output_tensors, output_masks, output_shapes + + return output_tensors, output_masks def _is_tensor_or_tensor_list(v): @@ -2292,11 +2354,24 @@ def _collect_previous_mask(input_tensors): PER_GRAPH_LAYER_NAME_UIDS = weakref.WeakKeyDictionary() -def _unique_layer_name(name): +def _get_default_graph_uid_map(): + graph = ops.get_default_graph() + name_uid_map = PER_GRAPH_LAYER_NAME_UIDS.get(graph, None) + if name_uid_map is None: + name_uid_map = collections.defaultdict(int) + PER_GRAPH_LAYER_NAME_UIDS[graph] = name_uid_map + return name_uid_map + + +def _unique_layer_name(name, name_uid_map=None, avoid_names=None): """Makes a layer name (or arbitrary string) unique within a TensorFlow graph. Arguments: name: String name to make unique. + name_uid_map: An optional defaultdict(int) to use when creating unique + names. If None (default), uses a per-Graph dictionary. + avoid_names: An optional set or dict with names which should not be used. If + None (default) does not avoid any names. Returns: Unique string name. @@ -2308,9 +2383,12 @@ def _unique_layer_name(name): _unique_layer_name('dense') # dense_2 ``` """ - graph = ops.get_default_graph() - if graph not in PER_GRAPH_LAYER_NAME_UIDS: - PER_GRAPH_LAYER_NAME_UIDS[graph] = collections.defaultdict(int) - layer_name_uids = PER_GRAPH_LAYER_NAME_UIDS[graph] - layer_name_uids[name] += 1 - return name + '_' + str(layer_name_uids[name]) + if name_uid_map is None: + name_uid_map = _get_default_graph_uid_map() + if avoid_names is None: + avoid_names = set() + proposed_name = None + while proposed_name is None or proposed_name in avoid_names: + name_uid_map[name] += 1 + proposed_name = name + '_' + str(name_uid_map[name]) + return proposed_name diff --git a/tensorflow/python/layers/base_test.py b/tensorflow/python/layers/base_test.py index 813a2fe755d0bc05c7db06546977ff6e05037e8f..71eff2f9657fde2855acfc602c54c6a38aedf5a3 100644 --- a/tensorflow/python/layers/base_test.py +++ b/tensorflow/python/layers/base_test.py @@ -20,6 +20,8 @@ from __future__ import print_function import copy +import numpy as np + from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -41,13 +43,13 @@ class BaseLayerTest(test.TestCase): @test_util.run_in_graph_and_eager_modes() def testLayerProperties(self): layer = base_layers.Layer(name='my_layer') - self.assertListEqual(layer.variables, []) - self.assertListEqual(layer.trainable_variables, []) - self.assertListEqual(layer.non_trainable_variables, []) + self.assertEqual(layer.variables, []) + self.assertEqual(layer.trainable_variables, []) + self.assertEqual(layer.non_trainable_variables, []) if context.in_graph_mode(): # updates, losses only suppported in GRAPH mode - self.assertListEqual(layer.updates, []) - self.assertListEqual(layer.losses, []) + self.assertEqual(layer.updates, []) + self.assertEqual(layer.losses, []) self.assertEqual(layer.built, False) layer = base_layers.Layer(name='my_layer', trainable=False) self.assertEqual(layer.trainable, False) @@ -60,11 +62,11 @@ class BaseLayerTest(test.TestCase): variable = layer.add_variable( 'my_var', [2, 2], initializer=init_ops.zeros_initializer()) self.assertEqual(variable.name, 'my_layer/my_var:0') - self.assertListEqual(layer.variables, [variable]) - self.assertListEqual(layer.trainable_variables, [variable]) - self.assertListEqual(layer.non_trainable_variables, []) + self.assertEqual(layer.variables, [variable]) + self.assertEqual(layer.trainable_variables, [variable]) + self.assertEqual(layer.non_trainable_variables, []) if context.in_graph_mode(): - self.assertListEqual( + self.assertEqual( layer.variables, ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) @@ -74,9 +76,9 @@ class BaseLayerTest(test.TestCase): 'non_trainable_var', [2, 2], initializer=init_ops.zeros_initializer(), trainable=False) - self.assertListEqual(layer.variables, [variable, variable_2]) - self.assertListEqual(layer.trainable_variables, [variable]) - self.assertListEqual(layer.non_trainable_variables, [variable_2]) + self.assertEqual(layer.variables, [variable, variable_2]) + self.assertEqual(layer.trainable_variables, [variable]) + self.assertEqual(layer.non_trainable_variables, [variable_2]) if context.in_graph_mode(): self.assertEqual( len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 1) @@ -105,8 +107,8 @@ class BaseLayerTest(test.TestCase): inputs = random_ops.random_uniform((5,), seed=1) layer.apply(inputs) layer.apply(inputs) - self.assertListEqual([v.name for v in layer.variables], - ['my_layer/my_var:0']) + self.assertEqual([v.name for v in layer.variables], + ['my_layer/my_var:0']) # Creating a layer with no scope leads to lazy construction of # the scope at apply() time. It uses scope "/base_name" @@ -120,7 +122,7 @@ class BaseLayerTest(test.TestCase): # The variables were created outside of the Layer, and # reuse=True, so the Layer does not own them and they are not # stored in its collection. - self.assertListEqual(lazy_layer.variables, []) + self.assertEqual(lazy_layer.variables, []) self.assertEqual(lazy_layer._scope.name, 'new_scope/my_layer') # Creating a layer with no scope leads to lazy construction of @@ -135,7 +137,7 @@ class BaseLayerTest(test.TestCase): # The variables were created outside of the Layer, and # reuse=True, so the Layer does not own them and they are not # stored in its collection. - self.assertListEqual(lazy_layer.variables, []) + self.assertEqual(lazy_layer.variables, []) self.assertEqual(lazy_layer._scope.name, 'new_scope') # Checking for graph equality is only done in GRAPH mode. @@ -183,14 +185,14 @@ class BaseLayerTest(test.TestCase): outputs = layer.apply(inputs) self.assertEqual(layer.built, True) self.assertEqual(outputs.op.name, 'my_layer/add') - self.assertListEqual([v.name - for v in layer.variables], ['my_layer/my_var:0']) + self.assertEqual([v.name + for v in layer.variables], ['my_layer/my_var:0']) with self.assertRaisesRegexp(ValueError, 'my_layer/this_will_break_on_second_call'): layer.apply(inputs) # The list of variables hasn't changed. - self.assertListEqual([v.name - for v in layer.variables], ['my_layer/my_var:0']) + self.assertEqual([v.name + for v in layer.variables], ['my_layer/my_var:0']) @test_util.run_in_graph_and_eager_modes() def testDeepCopy(self): @@ -435,8 +437,8 @@ class BaseLayerTest(test.TestCase): dense_layer.add_update(0, inputs=a) dense_layer.add_update(1, inputs=None) - self.assertListEqual(dense_layer.get_updates_for(a), [0]) - self.assertListEqual(dense_layer.get_updates_for(None), [1]) + self.assertEqual(dense_layer.get_updates_for(a), [0]) + self.assertEqual(dense_layer.get_updates_for(None), [1]) def test_get_losses_for(self): a = base_layers.Input(shape=(2,)) @@ -444,8 +446,8 @@ class BaseLayerTest(test.TestCase): dense_layer.add_loss(0, inputs=a) dense_layer.add_loss(1, inputs=None) - self.assertListEqual(dense_layer.get_losses_for(a), [0]) - self.assertListEqual(dense_layer.get_losses_for(None), [1]) + self.assertEqual(dense_layer.get_losses_for(a), [0]) + self.assertEqual(dense_layer.get_losses_for(None), [1]) def testTopologicalAttributes(self): # test layer attributes / methods related to cross-layer connectivity. @@ -612,7 +614,7 @@ class NetworkTest(test.TestCase): a = base_layers.Input(shape=(32,), name='input_a') b = base_layers.Input(shape=(32,), name='input_b') - self.assertListEqual(a.get_shape().as_list(), [None, 32]) + self.assertEqual(a.get_shape().as_list(), [None, 32]) a_layer, a_node_index, a_tensor_index = a._keras_history b_layer, _, _ = b._keras_history self.assertEqual(len(a_layer._inbound_nodes), 1) @@ -620,11 +622,11 @@ class NetworkTest(test.TestCase): node = a_layer._inbound_nodes[a_node_index] self.assertEqual(node.outbound_layer, a_layer) - self.assertListEqual(node.inbound_layers, []) - self.assertListEqual(node.input_tensors, [a]) - self.assertListEqual(node.input_shapes, [(None, 32)]) - self.assertListEqual(node.output_tensors, [a]) - self.assertListEqual(node.output_shapes, [(None, 32)]) + self.assertEqual(node.inbound_layers, []) + self.assertEqual(node.input_tensors, [a]) + self.assertEqual(node.input_shapes, [(None, 32)]) + self.assertEqual(node.output_tensors, [a]) + self.assertEqual(node.output_shapes, [(None, 32)]) dense = core_layers.Dense(16, name='dense_1') dense(a) @@ -632,12 +634,12 @@ class NetworkTest(test.TestCase): self.assertEqual(len(dense._inbound_nodes), 2) self.assertEqual(len(dense._outbound_nodes), 0) - self.assertListEqual(dense._inbound_nodes[0].inbound_layers, [a_layer]) + self.assertEqual(dense._inbound_nodes[0].inbound_layers, [a_layer]) self.assertEqual(dense._inbound_nodes[0].outbound_layer, dense) - self.assertListEqual(dense._inbound_nodes[1].inbound_layers, [b_layer]) + self.assertEqual(dense._inbound_nodes[1].inbound_layers, [b_layer]) self.assertEqual(dense._inbound_nodes[1].outbound_layer, dense) - self.assertListEqual(dense._inbound_nodes[0].input_tensors, [a]) - self.assertListEqual(dense._inbound_nodes[1].input_tensors, [b]) + self.assertEqual(dense._inbound_nodes[0].input_tensors, [a]) + self.assertEqual(dense._inbound_nodes[1].input_tensors, [b]) # Test config config_0 = dense._inbound_nodes[0].get_config() @@ -889,5 +891,67 @@ class NetworkTest(test.TestCase): self.assertAllEqual(self.evaluate(a * mask), self.evaluate(b)) +class DeferredModeTest(test.TestCase): + + def testDeferredTensorAttributes(self): + x = base_layers._DeferredTensor(shape=(None, 2), dtype='float32', name='x') + self.assertEqual(str(x), + 'DeferredTensor(\'x\', shape=(?, 2), dtype=float32)') + self.assertEqual(repr(x), + '<_DeferredTensor \'x\' shape=(?, 2) dtype=float32>') + + @test_util.run_in_graph_and_eager_modes() + def testSimpleNetworkBuilding(self): + inputs = base_layers.Input(shape=(32,)) + if context.in_eager_mode(): + self.assertIsInstance(inputs, base_layers._DeferredTensor) + self.assertEqual(inputs.dtype.name, 'float32') + self.assertEqual(inputs.shape.as_list(), [None, 32]) + + x = core_layers.Dense(2)(inputs) + if context.in_eager_mode(): + self.assertIsInstance(x, base_layers._DeferredTensor) + self.assertEqual(x.dtype.name, 'float32') + self.assertEqual(x.shape.as_list(), [None, 2]) + + outputs = core_layers.Dense(4)(x) + network = base_layers.Network(inputs, outputs) + self.assertIsInstance(network, base_layers.Network) + + if context.in_eager_mode(): + # It should be possible to call such a network on EagerTensors. + inputs = constant_op.constant( + np.random.random((10, 32)).astype('float32')) + outputs = network(inputs) + self.assertEqual(outputs.shape.as_list(), [10, 4]) + + @test_util.run_in_graph_and_eager_modes() + def testMultiIONetworkbuilding(self): + input_a = base_layers.Input(shape=(32,)) + input_b = base_layers.Input(shape=(16,)) + a = core_layers.Dense(16)(input_a) + + class AddLayer(base_layers.Layer): + + def call(self, inputs): + return inputs[0] + inputs[1] + + def _compute_output_shape(self, input_shape): + return input_shape[0] + + c = AddLayer()([a, input_b]) # pylint: disable=not-callable + c = core_layers.Dense(2)(c) + + network = base_layers.Network([input_a, input_b], [a, c]) + if context.in_eager_mode(): + a_val = constant_op.constant( + np.random.random((10, 32)).astype('float32')) + b_val = constant_op.constant( + np.random.random((10, 16)).astype('float32')) + outputs = network([a_val, b_val]) + self.assertEqual(len(outputs), 2) + self.assertEqual(outputs[0].shape.as_list(), [10, 16]) + self.assertEqual(outputs[1].shape.as_list(), [10, 2]) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py index 9850cd33b01a7b9aeb4d72bfb9808a41654d3d80..0c7ce0283544059aa0bab8f9d79512867ab531fb 100644 --- a/tensorflow/python/layers/convolutional.py +++ b/tensorflow/python/layers/convolutional.py @@ -153,22 +153,18 @@ class _Conv(base.Layer): self.bias = None self.input_spec = base.InputSpec(ndim=self.rank + 2, axes={channel_axis: input_dim}) - with ops.name_scope(None, 'convolution', [self.kernel]) as name: - self._convolution_op = nn_ops.Convolution( - input_shape, - filter_shape=self.kernel.get_shape(), - dilation_rate=self.dilation_rate, - strides=self.strides, - padding=self.padding.upper(), - data_format=utils.convert_data_format(self.data_format, - self.rank + 2), - name=name) + self._convolution_op = nn_ops.Convolution( + input_shape, + filter_shape=self.kernel.get_shape(), + dilation_rate=self.dilation_rate, + strides=self.strides, + padding=self.padding.upper(), + data_format=utils.convert_data_format(self.data_format, + self.rank + 2)) self.built = True def call(self, inputs): - # TODO(agarwal): do we need this name_scope ? - with ops.name_scope(None, 'convolution', [inputs, self.kernel]): - outputs = self._convolution_op(inputs, self.kernel) + outputs = self._convolution_op(inputs, self.kernel) if self.use_bias: if self.data_format == 'channels_first': @@ -387,6 +383,9 @@ def conv1d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Conv1D( filters=filters, @@ -587,6 +586,9 @@ def conv2d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Conv2D( filters=filters, @@ -789,6 +791,9 @@ def conv3d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Conv3D( filters=filters, @@ -1108,6 +1113,9 @@ def separable_conv2d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = SeparableConv2D( filters=filters, @@ -1403,6 +1411,9 @@ def conv2d_transpose(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Conv2DTranspose( filters=filters, @@ -1714,6 +1725,9 @@ def conv3d_transpose(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Conv3DTranspose( filters=filters, diff --git a/tensorflow/python/layers/core.py b/tensorflow/python/layers/core.py index ef9ff5790c80efbb79a5bb2078c2b013ee5fb789..76e8fbef2f4b187acbbf094f5a3b880341cbdd61 100644 --- a/tensorflow/python/layers/core.py +++ b/tensorflow/python/layers/core.py @@ -231,6 +231,9 @@ def dense( Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Dense(units, activation=activation, @@ -333,6 +336,9 @@ def dropout(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = Dropout(rate, noise_shape=noise_shape, seed=seed, name=name) return layer.apply(inputs, training=training) diff --git a/tensorflow/python/layers/core_test.py b/tensorflow/python/layers/core_test.py index d917dcb69c7c33f23321320604fedc6accb779ba..b67df89f81fafb1d3df9b2caba15efa2b96d9e2f 100644 --- a/tensorflow/python/layers/core_test.py +++ b/tensorflow/python/layers/core_test.py @@ -23,6 +23,7 @@ import collections import numpy as np from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape @@ -203,21 +204,15 @@ class DenseTest(test.TestCase): self.assertEqual(len(loss_keys), 1) self.assertListEqual(dense.losses, loss_keys) - @test_util.run_in_graph_and_eager_modes() def testFunctionalDense(self): - inputs = random_ops.random_uniform((5, 3), seed=1) - outputs = core_layers.dense( - inputs, 2, activation=nn_ops.relu, name='my_dense') - if context.in_graph_mode(): + with self.test_session(): + inputs = random_ops.random_uniform((5, 3), seed=1) + outputs = core_layers.dense( + inputs, 2, activation=nn_ops.relu, name='my_dense') self.assertEqual( len(ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)), 2) self.assertEqual(outputs.op.name, 'my_dense/Relu') - else: - self.assertEqual( - len(_get_variable_dict_from_varstore().values()), 2) - self.assertEqual(outputs.get_shape().as_list(), [5, 2]) - @test_util.run_in_graph_and_eager_modes() def testFunctionalDenseTwice(self): inputs = random_ops.random_uniform((5, 3), seed=1) core_layers.dense(inputs, 2) @@ -249,25 +244,38 @@ class DenseTest(test.TestCase): vars2 = variables.trainable_variables() self.assertEqual(vars1, vars2) - @test_util.run_in_graph_and_eager_modes() def testFunctionalDenseInitializerFromScope(self): with variable_scope.variable_scope( - 'scope', initializer=init_ops.ones_initializer()): + 'scope', initializer=init_ops.ones_initializer()), self.test_session(): inputs = random_ops.random_uniform((5, 3), seed=1) core_layers.dense(inputs, 2) - self.evaluate(variables.global_variables_initializer()) + variables.global_variables_initializer().run() weights = _get_variable_dict_from_varstore() self.assertEqual(len(weights), 2) # Check that the matrix weights got initialized to ones (from scope). - self.assertAllClose( - self.evaluate(weights['scope/dense/kernel'].read_value()), - np.ones((3, 2))) + self.assertAllClose(weights['scope/dense/kernel'].read_value().eval(), + np.ones((3, 2))) # Check that the bias still got initialized to zeros. - self.assertAllClose( - self.evaluate(weights['scope/dense/bias'].read_value()), - np.zeros((2))) + self.assertAllClose(weights['scope/dense/bias'].read_value().eval(), + np.zeros((2))) + + def testEagerExecution(self): + with context.eager_mode(): + container = variable_scope.EagerVariableStore() + x = constant_op.constant([[2.0]]) + with container.as_default(): + y = core_layers.dense( + x, 1, name='my_dense', + kernel_initializer=init_ops.ones_initializer()) + self.assertAllEqual(y, [[2.0]]) + self.assertEqual(len(container.variables()), 2) + # Recreate the layer to test reuse. + with container.as_default(): + core_layers.dense( + x, 1, name='my_dense', + kernel_initializer=init_ops.ones_initializer()) + self.assertEqual(len(container.variables()), 2) - @test_util.run_in_graph_and_eager_modes() def testFunctionalDenseWithCustomGetter(self): called = [0] @@ -280,26 +288,26 @@ class DenseTest(test.TestCase): core_layers.dense(inputs, 2) self.assertEqual(called[0], 2) - @test_util.run_in_graph_and_eager_modes() def testFunctionalDenseInScope(self): - with variable_scope.variable_scope('test'): - inputs = random_ops.random_uniform((5, 3), seed=1) - core_layers.dense(inputs, 2, name='my_dense') - var_dict = _get_variable_dict_from_varstore() - var_key = 'test/my_dense/kernel' - self.assertEqual(var_dict[var_key].name, '%s:0' % var_key) - with variable_scope.variable_scope('test1') as scope: - inputs = random_ops.random_uniform((5, 3), seed=1) - core_layers.dense(inputs, 2, name=scope) - var_dict = _get_variable_dict_from_varstore() - var_key = 'test1/kernel' - self.assertEqual(var_dict[var_key].name, '%s:0' % var_key) - with variable_scope.variable_scope('test2'): - inputs = random_ops.random_uniform((5, 3), seed=1) - core_layers.dense(inputs, 2) - var_dict = _get_variable_dict_from_varstore() - var_key = 'test2/dense/kernel' - self.assertEqual(var_dict[var_key].name, '%s:0' % var_key) + with self.test_session(): + with variable_scope.variable_scope('test'): + inputs = random_ops.random_uniform((5, 3), seed=1) + core_layers.dense(inputs, 2, name='my_dense') + var_dict = _get_variable_dict_from_varstore() + var_key = 'test/my_dense/kernel' + self.assertEqual(var_dict[var_key].name, '%s:0' % var_key) + with variable_scope.variable_scope('test1') as scope: + inputs = random_ops.random_uniform((5, 3), seed=1) + core_layers.dense(inputs, 2, name=scope) + var_dict = _get_variable_dict_from_varstore() + var_key = 'test1/kernel' + self.assertEqual(var_dict[var_key].name, '%s:0' % var_key) + with variable_scope.variable_scope('test2'): + inputs = random_ops.random_uniform((5, 3), seed=1) + core_layers.dense(inputs, 2) + var_dict = _get_variable_dict_from_varstore() + var_key = 'test2/dense/kernel' + self.assertEqual(var_dict[var_key].name, '%s:0' % var_key) @test_util.run_in_graph_and_eager_modes() def testComputeOutputShape(self): @@ -389,17 +397,16 @@ class DropoutTest(test.TestCase): self.assertAlmostEqual(0., np_output.min()) self.assertAllClose(np_output[:, 0, :], np_output[:, 1, :]) - @test_util.run_in_graph_and_eager_modes() def testFunctionalDropout(self): - inputs = array_ops.ones((5, 5)) - dropped = core_layers.dropout(inputs, 0.5, training=True, seed=1) - if context.in_graph_mode(): - self.evaluate(variables.global_variables_initializer()) - np_output = self.evaluate(dropped) - self.assertAlmostEqual(0., np_output.min()) - dropped = core_layers.dropout(inputs, 0.5, training=False, seed=1) - np_output = self.evaluate(dropped) - self.assertAllClose(np.ones((5, 5)), np_output) + with self.test_session(): + inputs = array_ops.ones((5, 5)) + dropped = core_layers.dropout(inputs, 0.5, training=True, seed=1) + variables.global_variables_initializer().run() + np_output = self.evaluate(dropped) + self.assertAlmostEqual(0., np_output.min()) + dropped = core_layers.dropout(inputs, 0.5, training=False, seed=1) + np_output = self.evaluate(dropped) + self.assertAllClose(np.ones((5, 5)), np_output) def testDynamicRate(self): with self.test_session() as sess: diff --git a/tensorflow/python/layers/maxout.py b/tensorflow/python/layers/maxout.py index 1ea36dbf6a232cae1ebd15ae56b8be30edb210f0..ed048845a0b88344b357836a838231677cbf40ce 100644 --- a/tensorflow/python/layers/maxout.py +++ b/tensorflow/python/layers/maxout.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import gen_array_ops diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py index df2b97f03efcc031479d41ff67718cbfe28051f3..01f56abc70ef52eda25a4b247ae9b536b60266d5 100644 --- a/tensorflow/python/layers/normalization.py +++ b/tensorflow/python/layers/normalization.py @@ -101,6 +101,18 @@ class BatchNormalization(base.Layer): Normalization", which creates virtual sub-batches which are each normalized separately (with shared gamma, beta, and moving statistics). Must divide the actual batch size during execution. + adjustment: A function taking the `Tensor` containing the (dynamic) shape of + the input tensor and returning a pair (scale, bias) to apply to the + normalized values (before gamma and beta), only during training. For + example, if axis==-1, + `adjustment = lambda shape: ( + tf.random_uniform(shape[-1:], 0.93, 1.07), + tf.random_uniform(shape[-1:], -0.1, 0.1))` + will scale the normalized value by up to 7% up or down, then shift the + result by up to 0.1 (with independent scaling and bias for each feature + but shared across all examples), and finally apply gamma and/or beta. If + `None`, no adjustment is applied. Cannot be specified if + virtual_batch_size is specified. name: A string, the name of the layer. """ @@ -124,6 +136,7 @@ class BatchNormalization(base.Layer): fused=None, trainable=True, virtual_batch_size=None, + adjustment=None, name=None, **kwargs): super(BatchNormalization, self).__init__( @@ -143,6 +156,7 @@ class BatchNormalization(base.Layer): self.gamma_constraint = gamma_constraint self.renorm = renorm self.virtual_batch_size = virtual_batch_size + self.adjustment = adjustment if fused is None: fused = True @@ -192,6 +206,9 @@ class BatchNormalization(base.Layer): if 0 in self.axis: raise ValueError('When using virtual_batch_size, the batch dimension ' 'must be 0 and thus axis cannot include 0') + if self.adjustment is not None: + raise ValueError('When using virtual_batch_size, adjustment cannot ' + 'be specified') if self.fused: # Currently fused batch norm doesn't support renorm and beta/gamma @@ -204,7 +221,8 @@ class BatchNormalization(base.Layer): self.axis in [[1], [3]] and self.beta_regularizer is None and self.gamma_regularizer is None and - self.virtual_batch_size is None) + self.virtual_batch_size is None and + self.adjustment is None) # TODO(chrisying): fused batch norm is currently not supported for # multi-axis batch norm and by extension virtual batches. In some cases, # it might be possible to use fused batch norm but would require reshaping @@ -418,27 +436,30 @@ class BatchNormalization(base.Layer): if dmax is not None: d = math_ops.maximum(d, -dmax) d = math_ops.minimum(d, dmax) - # When not training, use r=1, d=0, and decay=1 meaning no updates. + # When not training, use r=1, d=0. r = utils.smart_cond(training, lambda: r, lambda: array_ops.ones_like(r)) d = utils.smart_cond(training, lambda: d, lambda: array_ops.zeros_like(d)) - decay = utils.smart_cond(training, lambda: self.renorm_momentum, lambda: 1.) def _update_renorm_variable(var, weight, value): """Updates a moving average and weight, returns the unbiased value.""" - # Update the variables without zero debiasing. The debiasing will be - # accomplished by dividing the exponential moving average by the weight. - # For example, after a single update, the moving average would be - # (1-decay) * value. and the weight will be 1-decay, with their ratio - # giving value. - # Make sure the weight is not updated until before r and d computation. value = array_ops.identity(value) - with ops.control_dependencies([value]): - weight_value = array_ops.constant(1., dtype=weight.dtype) - new_var = moving_averages.assign_moving_average( - var, value, decay, zero_debias=False) - new_weight = moving_averages.assign_moving_average( - weight, weight_value, decay, zero_debias=False) - return new_var / new_weight + def _do_update(): + # Update the variables without zero debiasing. The debiasing will be + # accomplished by dividing the exponential moving average by the weight. + # For example, after a single update, the moving average would be + # (1-decay) * value. and the weight will be 1-decay, with their ratio + # giving the value. + # Make sure the weight is not updated until before r and d computation. + with ops.control_dependencies([value]): + weight_value = array_ops.constant(1., dtype=weight.dtype) + new_var = moving_averages.assign_moving_average( + var, value, self.renorm_momentum, zero_debias=False) + new_weight = moving_averages.assign_moving_average( + weight, weight_value, self.renorm_momentum, zero_debias=False) + return new_var / new_weight + def _fake_update(): + return array_ops.identity(var) + return utils.smart_cond(training, _do_update, _fake_update) with ops.colocate_with(self.moving_mean): new_mean = _update_renorm_variable(self.renorm_mean, @@ -482,11 +503,41 @@ class BatchNormalization(base.Layer): if self.virtual_batch_size is not None: del reduction_axes[1] # Do not reduce along virtual batch dim - scale, offset = self.gamma, self.beta + # Broadcasting only necessary for single-axis batch norm where the axis is + # not the last dimension + broadcast_shape = [1] * ndims + broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value + def _broadcast(v): + if (v is not None and + len(v.get_shape()) != ndims and + reduction_axes != list(range(ndims - 1))): + return array_ops.reshape(v, broadcast_shape) + return v + + scale, offset = _broadcast(self.gamma), _broadcast(self.beta) + + def _compose_transforms(scale, offset, then_scale, then_offset): + if then_scale is not None: + scale *= then_scale + offset *= then_scale + if then_offset is not None: + offset += then_offset + return (scale, offset) # Determine a boolean value for `training`: could be True, False, or None. training_value = utils.constant_value(training) if training_value is not False: + if self.adjustment: + adj_scale, adj_bias = self.adjustment(array_ops.shape(inputs)) + # Adjust only during training. + adj_scale = utils.smart_cond(training, + lambda: adj_scale, + lambda: array_ops.ones_like(adj_scale)) + adj_bias = utils.smart_cond(training, + lambda: adj_bias, + lambda: array_ops.zeros_like(adj_bias)) + scale, offset = _compose_transforms(adj_scale, adj_bias, scale, offset) + # Some of the computations here are not necessary when training==False # but not a constant. However, this makes the code simpler. keep_dims = self.virtual_batch_size is not None or len(self.axis) > 1 @@ -508,18 +559,12 @@ class BatchNormalization(base.Layer): # When training, the normalized values (say, x) will be transformed as # x * gamma + beta without renorm, and (x * r + d) * gamma + beta # = x * (r * gamma) + (d * gamma + beta) with renorm. - scale = array_ops.stop_gradient(r, name='renorm_r') - offset = array_ops.stop_gradient(d, name='renorm_d') - if self.gamma is not None: - scale *= self.gamma - offset *= self.gamma - if self.beta is not None: - offset += self.beta + r = _broadcast(array_ops.stop_gradient(r, name='renorm_r')) + d = _broadcast(array_ops.stop_gradient(d, name='renorm_d')) + scale, offset = _compose_transforms(r, d, scale, offset) else: new_mean, new_variance = mean, variance - # Update moving averages when training, and prevent updates otherwise. - decay = utils.smart_cond(training, lambda: self.momentum, lambda: 1.) if self.virtual_batch_size is not None: # This isn't strictly correct since in ghost batch norm, you are # supposed to sequentially update the moving_mean and moving_variance @@ -531,10 +576,18 @@ class BatchNormalization(base.Layer): new_variance = math_ops.reduce_mean(new_variance, axis=1, keep_dims=True) - mean_update = moving_averages.assign_moving_average( - self.moving_mean, new_mean, decay, zero_debias=False) - variance_update = moving_averages.assign_moving_average( - self.moving_variance, new_variance, decay, zero_debias=False) + def _do_update(var, value): + return moving_averages.assign_moving_average( + var, value, self.momentum, zero_debias=False) + + mean_update = utils.smart_cond( + training, + lambda: _do_update(self.moving_mean, new_mean), + lambda: self.moving_mean) + variance_update = utils.smart_cond( + training, + lambda: _do_update(self.moving_variance, new_variance), + lambda: self.moving_variance) if context.in_graph_mode(): self.add_update(mean_update, inputs=inputs) self.add_update(variance_update, inputs=inputs) @@ -542,24 +595,14 @@ class BatchNormalization(base.Layer): else: mean, variance = self.moving_mean, self.moving_variance - # Broadcasting only necessary for single-axis batch norm where the axis is - # not the last dimension - broadcast_shape = [1] * ndims - broadcast_shape[self.axis[0]] = input_shape[self.axis[0]].value - rank = len(inputs.get_shape()) - def _broadcast(v): - if (v is not None and - len(v.get_shape()) != rank and - reduction_axes != list(range(ndims))[:-1]): - return array_ops.reshape(v, broadcast_shape) - return v - outputs = nn.batch_normalization(inputs, _broadcast(mean), _broadcast(variance), - _broadcast(offset), - _broadcast(scale), + offset, + scale, self.epsilon) + # If some components of the shape got lost due to adjustments, fix that. + outputs.set_shape(input_shape) if self.virtual_batch_size is not None: return undo_virtual_batching(outputs) @@ -589,7 +632,8 @@ def batch_normalization(inputs, renorm_clipping=None, renorm_momentum=0.99, fused=None, - virtual_batch_size=None): + virtual_batch_size=None, + adjustment=None): """Functional interface for the batch normalization layer. Reference: http://arxiv.org/abs/1502.03167 @@ -667,9 +711,24 @@ def batch_normalization(inputs, Normalization", which creates virtual sub-batches which are each normalized separately (with shared gamma, beta, and moving statistics). Must divide the actual batch size during execution. + adjustment: A function taking the `Tensor` containing the (dynamic) shape of + the input tensor and returning a pair (scale, bias) to apply to the + normalized values (before gamma and beta), only during training. For + example, if axis==-1, + `adjustment = lambda shape: ( + tf.random_uniform(shape[-1:], 0.93, 1.07), + tf.random_uniform(shape[-1:], -0.1, 0.1))` + will scale the normalized value by up to 7% up or down, then shift the + result by up to 0.1 (with independent scaling and bias for each feature + but shared across all examples), and finally apply gamma and/or beta. If + `None`, no adjustment is applied. Cannot be specified if + virtual_batch_size is specified. Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = BatchNormalization( axis=axis, @@ -691,6 +750,7 @@ def batch_normalization(inputs, fused=fused, trainable=trainable, virtual_batch_size=virtual_batch_size, + adjustment=adjustment, name=name, _reuse=reuse, _scope=name) @@ -701,4 +761,3 @@ def batch_normalization(inputs, BatchNorm = BatchNormalization batch_norm = batch_normalization - diff --git a/tensorflow/python/layers/normalization_test.py b/tensorflow/python/layers/normalization_test.py index f8d9d2948ce013124a2737e5fb6c68d1a62563a5..90ebdc8c86f425c34a90204fbf4b8f2b8061ae4e 100644 --- a/tensorflow/python/layers/normalization_test.py +++ b/tensorflow/python/layers/normalization_test.py @@ -823,6 +823,112 @@ class BNTest(test.TestCase): self.assertAllClose(y_train, yt_val_train, atol=1e-5) self.assertAllClose(y_test, yt_val_test, atol=1e-5) + def testAdjustment(self): + shape = (4, 3) + xt = array_ops.placeholder(dtypes.float32, shape) + momentum = 0.99 + gamma = 2. + beta = 3. + epsilon = 0.001 + adjust_scale = random_ops.random_uniform(shape[-1:], 0.5, 1.5) + adjust_bias = random_ops.random_uniform(shape[-1:], -.2, .2) + bn = normalization_layers.BatchNormalization( + axis=1, + gamma_initializer=init_ops.constant_initializer(gamma), + beta_initializer=init_ops.constant_initializer(beta), + epsilon=epsilon, + momentum=momentum, + adjustment=lambda _: (adjust_scale, adjust_bias)) + training = array_ops.placeholder(dtypes.bool) + yt = bn.apply(xt, training=training) + + moving_mean = 0. + moving_variance = 1. + with self.test_session(use_gpu=True) as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(5): + x = np.random.random(shape) + yt_val_train, adj_scale_val, adj_bias_val = sess.run( + [yt, adjust_scale, adjust_bias] + bn.updates, + feed_dict={xt: x, training: True})[:3] + yt_val_test = sess.run([yt] + bn.updates, + feed_dict={xt: x, training: False})[0] + + mean = x.mean(0) + variance = x.var(0) + y_train = (((x - mean) / (variance + epsilon) ** 0.5) * adj_scale_val + + adj_bias_val) * gamma + beta + moving_mean += (mean - moving_mean) * (1. - momentum) + moving_variance += (variance - moving_variance) * (1. - momentum) + + y_test = ((x - moving_mean) / (moving_variance + epsilon) ** 0.5 * + gamma) + beta + + self.assertAllClose(y_train, yt_val_train, atol=1e-5) + self.assertAllClose(y_test, yt_val_test, atol=1e-5) + + def testRenormWithAdjustment(self): + shape = (4, 3) + xt = array_ops.placeholder(dtypes.float32, shape) + momentum = 0.99 + renorm_momentum = 0.8 + rmax = 1.1 + rmin = 0.9 + dmax = 0.1 + gamma = 2. + beta = 3. + epsilon = 0.001 + adjust_scale = random_ops.random_uniform(shape[-1:], 0.5, 1.5) + adjust_bias = random_ops.random_uniform(shape[-1:], -.2, .2) + bn = normalization_layers.BatchNormalization( + axis=1, + gamma_initializer=init_ops.constant_initializer(gamma), + beta_initializer=init_ops.constant_initializer(beta), + epsilon=epsilon, + momentum=momentum, + renorm=True, + renorm_clipping={'rmax': rmax, 'rmin': rmin, 'dmax': dmax}, + renorm_momentum=renorm_momentum, + adjustment=lambda _: (adjust_scale, adjust_bias)) + training = array_ops.placeholder(dtypes.bool) + yt = bn.apply(xt, training=training) + + moving_mean = 0. + moving_variance = 1. + renorm_mean = renorm_stddev = 0. + renorm_weight = 0. + with self.test_session(use_gpu=True) as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(5): + x = np.random.random(shape) + yt_val_train, adj_scale_val, adj_bias_val = sess.run( + [yt, adjust_scale, adjust_bias] + bn.updates, + feed_dict={xt: x, training: True})[:3] + yt_val_test = sess.run([yt] + bn.updates, + feed_dict={xt: x, training: False})[0] + + mean = x.mean(0) + stddev = np.sqrt(x.var(0) + epsilon) + adj_mean = renorm_mean + (1. - renorm_weight) * mean + adj_stddev = renorm_stddev + (1. - renorm_weight) * stddev + r = (stddev / adj_stddev).clip(rmin, rmax) + d = ((mean - adj_mean) / adj_stddev).clip(-dmax, dmax) + y_train = (((x - mean) / stddev * r + d) * adj_scale_val + + adj_bias_val) * gamma + beta + renorm_mean += (mean - renorm_mean) * (1. - renorm_momentum) + renorm_stddev += (stddev - renorm_stddev) * (1. - renorm_momentum) + renorm_weight += (1. - renorm_weight) * (1. - renorm_momentum) + moving_mean += (renorm_mean / renorm_weight - + moving_mean) * (1. - momentum) + moving_variance += ((renorm_stddev / renorm_weight) ** 2 - epsilon - + moving_variance) * (1. - momentum) + + y_test = ((x - moving_mean) / (moving_variance + epsilon) ** 0.5 * + gamma) + beta + + self.assertAllClose(y_train, yt_val_train, atol=1e-5) + self.assertAllClose(y_test, yt_val_test, atol=1e-5) + def testGhostBNNegativeVirtualBatch(self): shape = [6, 5, 4, 3] inp = random_ops.random_uniform(shape, seed=1) diff --git a/tensorflow/python/layers/pooling.py b/tensorflow/python/layers/pooling.py index 6245ec505486feebc1c944737a9f70688a0a98a7..78dd617bec85cc29c93a86df3601f2accd5c240a 100644 --- a/tensorflow/python/layers/pooling.py +++ b/tensorflow/python/layers/pooling.py @@ -20,6 +20,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import tensor_shape from tensorflow.python.layers import base from tensorflow.python.layers import utils @@ -144,6 +145,9 @@ def average_pooling1d(inputs, pool_size, strides, Returns: The output tensor, of rank 3. + + Raises: + ValueError: if eager execution is enabled. """ layer = AveragePooling1D(pool_size=pool_size, strides=strides, @@ -206,6 +210,9 @@ def max_pooling1d(inputs, pool_size, strides, Returns: The output tensor, of rank 3. + + Raises: + ValueError: if eager execution is enabled. """ layer = MaxPooling1D(pool_size=pool_size, strides=strides, @@ -344,6 +351,9 @@ def average_pooling2d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = AveragePooling2D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, @@ -409,6 +419,9 @@ def max_pooling2d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = MaxPooling2D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, @@ -560,6 +573,9 @@ def average_pooling3d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = AveragePooling3D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, @@ -629,6 +645,9 @@ def max_pooling3d(inputs, Returns: Output tensor. + + Raises: + ValueError: if eager execution is enabled. """ layer = MaxPooling3D(pool_size=pool_size, strides=strides, padding=padding, data_format=data_format, diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc index 84cb4885f6e68617c1bc1ea10a76504976315b26..a62847614c6d230a7c65a6f461187f1a170613cd 100644 --- a/tensorflow/python/lib/core/py_func.cc +++ b/tensorflow/python/lib/core/py_func.cc @@ -297,8 +297,15 @@ Status ConvertNdarrayToTensor(PyObject* obj, Tensor* ret) { char* el; Py_ssize_t el_size; if (PyBytes_AsStringAndSize(input_data[i], &el, &el_size) == -1) { - return errors::Unimplemented("Unsupported object type ", - input_data[i]->ob_type->tp_name); +#if PY_MAJOR_VERSION >= 3 + el = PyUnicode_AsUTF8AndSize(input_data[i], &el_size); + if (!el) { +#endif + return errors::Unimplemented("Unsupported object type ", + input_data[i]->ob_type->tp_name); +#if PY_MAJOR_VERSION >= 3 + } +#endif } tflat(i) = string(el, el_size); } diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py index 9f8acb2ae38698dc39794700a97aac3826130395..2ee298ad44e6ea12a204779a4e2eec68015a2d3a 100644 --- a/tensorflow/python/ops/array_grad.py +++ b/tensorflow/python/ops/array_grad.py @@ -397,7 +397,7 @@ def _GatherV2Grad(op, grad): # For axis 0 gathers, build an appropriately shaped IndexedSlices. if axis_static == 0: if context.in_eager_mode(): - params_tail_shape = params_shape.as_cpu_tensor()[1:] + params_tail_shape = params_shape.cpu()[1:] else: params_tail_shape = params_shape[1:] values_shape = array_ops.concat([indices_size, params_tail_shape], 0) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index c00efb16ba0ba025570c94cbea4ba539e638051e..e783fc29ebfdab0fa6f7c70f529ab1f9e7cd0958 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -66,6 +66,7 @@ See the @{$python/array_ops} guide. @@one_hot @@sequence_mask @@dequantize +@@quantize @@quantize_v2 @@quantized_concat @@setdiff1d @@ -309,8 +310,8 @@ def size(input, name=None, out_type=dtypes.int32): # pylint: disable=redefined-builtin """Returns the size of a tensor. - This operation returns an integer representing the number of elements in - `input`. + Returns a 0-D `Tensor` representing the number of elements in `input` + of type `out_type`. Defaults to tf.int32. For example: @@ -322,11 +323,15 @@ def size(input, name=None, out_type=dtypes.int32): Args: input: A `Tensor` or `SparseTensor`. name: A name for the operation (optional). - out_type: (Optional) The specified output type of the operation - (`int32` or `int64`). Defaults to tf.int32. + out_type: (Optional) The specified non-quantized numeric output type + of the operation. Defaults to `tf.int32`. Returns: - A `Tensor` of type `out_type`. Defaults to tf.int32. + A `Tensor` of type `out_type`. Defaults to `tf.int32`. + + @compatibility(numpy) + Equivalent to np.size() + @end_compatibility """ return size_internal(input, name, optimize=True, out_type=out_type) @@ -339,11 +344,11 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32): input: A `Tensor` or `SparseTensor`. name: A name for the operation (optional). optimize: if true, encode the size as a constant when possible. - out_type: (Optional) The specified output type of the operation - (`int32` or `int64`). Defaults to tf.int32. + out_type: (Optional) The specified non-quantized numeric output type + of the operation. Defaults to `tf.int32`. Returns: - A `Tensor` of type `out_type`. + A `Tensor` of type `out_type`. Defaults to `tf.int32`. """ with ops.name_scope(name, "Size", [input]) as name: if isinstance(input, (sparse_tensor.SparseTensor, @@ -2466,14 +2471,9 @@ def where(condition, x=None, y=None, name=None): """ if x is None and y is None: with ops.name_scope(name, "Where", [condition]) as name: - # Temporarily create an old style WhereOp nodedef + Operation without the - # attribute "T". - # TODO(b/67720963): Roll this back when the issue is resolved. - condition = gen_math_ops.cast(condition, dtypes.bool) - output = gen_array_ops.where(input=condition, name=name) - if context.in_graph_mode(): - output.op._node_def.attr.clear() - return output + condition = ops.convert_to_tensor( + condition, preferred_dtype=dtypes.bool, name="condition") + return gen_array_ops.where(input=condition, name=name) elif x is not None and y is not None: return gen_math_ops._select(condition=condition, t=x, e=y, name=name) else: @@ -2526,3 +2526,49 @@ def gather(params, indices, validate_indices=None, name=None, axis=0): gather.__doc__ = gen_array_ops.gather_v2.__doc__ + + +# Define quantize_v2 here in order to make name the second-to-last attribute, +# because round_mode was added later. +@deprecation.deprecated( + "2017-10-25", + "`tf.quantize_v2` is deprecated, please use `tf.quantize` instead.") +def quantize_v2(input, # pylint: disable=redefined-builtin + min_range, + max_range, + T, + mode="MIN_COMBINED", + name=None, + round_mode="HALF_AWAY_FROM_ZERO"): + return gen_array_ops.quantize_v2(input, + min_range, + max_range, + T=T, + mode=mode, + name=name, + round_mode=round_mode) + + +quantize_v2.__doc__ = """Please use `tf.quantize` instead.""" + + +# We want to expose tf.quantize instead of tf.quantize_v2; we can deprecate +# tf.quantize_v2 in next version of TensorFlow. +def quantize(input, # pylint: disable=redefined-builtin + min_range, + max_range, + T, + mode="MIN_COMBINED", + round_mode="HALF_AWAY_FROM_ZERO", + name=None): + return gen_array_ops.quantize_v2( + input, + min_range, + max_range, + T, + mode=mode, + round_mode=round_mode, + name=name) + + +quantize.__doc__ = gen_array_ops.quantize_v2.__doc__ diff --git a/tensorflow/python/ops/bitwise_ops.py b/tensorflow/python/ops/bitwise_ops.py index 44daf1353706f9c91679a7cd3d8a6ea77d17b879..e8e187e68f92d94b20e5e6ee0c707ea33a5e2f43 100644 --- a/tensorflow/python/ops/bitwise_ops.py +++ b/tensorflow/python/ops/bitwise_ops.py @@ -19,6 +19,8 @@ @@bitwise_or @@bitwise_xor @@invert +@@left_shift +@@right_shift """ from __future__ import absolute_import @@ -37,5 +39,7 @@ ops.NotDifferentiable("BitwiseOr") ops.NotDifferentiable("BitwiseXor") ops.NotDifferentiable("Invert") ops.NotDifferentiable("PopulationCount") +ops.NotDifferentiable("LeftShift") +ops.NotDifferentiable("RightShift") remove_undocumented(__name__) diff --git a/tensorflow/python/ops/bitwise_ops_test.py b/tensorflow/python/ops/bitwise_ops_test.py index 1d08c8f82dcb01e9d4c386b6a5033bf9628014ec..fa1b219b1771dbd8f99939d8f6571d2a8791433e 100644 --- a/tensorflow/python/ops/bitwise_ops_test.py +++ b/tensorflow/python/ops/bitwise_ops_test.py @@ -93,5 +93,47 @@ class BitwiseOpTest(test_util.TensorFlowTestCase): expected = [dtype.max - x for x in inputs] self.assertAllEqual(inverted, expected) + def testShiftsWithPositiveLHS(self): + dtype_list = [np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64] + + with self.test_session(use_gpu=True) as sess: + for dtype in dtype_list: + lhs = np.array([0, 5, 3, 14], dtype=dtype) + rhs = np.array([5, 0, 7, 3], dtype=dtype) + left_shift_result, right_shift_result = sess.run( + [bitwise_ops.left_shift(lhs, rhs), + bitwise_ops.right_shift(lhs, rhs)]) + self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs)) + self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs)) + + def testShiftsWithNegativeLHS(self): + dtype_list = [np.int8, np.int16, np.int32, np.int64] + + with self.test_session(use_gpu=True) as sess: + for dtype in dtype_list: + lhs = np.array([-1, -5, -3, -14], dtype=dtype) + rhs = np.array([5, 0, 7, 11], dtype=dtype) + left_shift_result, right_shift_result = sess.run( + [bitwise_ops.left_shift(lhs, rhs), + bitwise_ops.right_shift(lhs, rhs)]) + self.assertAllEqual(left_shift_result, np.left_shift(lhs, rhs)) + self.assertAllEqual(right_shift_result, np.right_shift(lhs, rhs)) + + def testImplementationDefinedShiftsDoNotCrash(self): + dtype_list = [np.int8, np.int16, np.int32, np.int64] + + with self.test_session(use_gpu=True) as sess: + for dtype in dtype_list: + lhs = np.array([-1, -5, -3, -14], dtype=dtype) + rhs = np.array([-2, 64, 101, 32], dtype=dtype) + # We intentionally do not test for specific values here since the exact + # outputs are implementation-defined. However, we should not crash or + # trigger an undefined-behavior error from tools such as + # AddressSanitizer. + sess.run([bitwise_ops.left_shift(lhs, rhs), + bitwise_ops.right_shift(lhs, rhs)]) + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py index fb4817528551f6a1b0c8be33c8508beec8a83693..ceee009104c8ac0d87795cf9d594914e899a921b 100644 --- a/tensorflow/python/ops/check_ops.py +++ b/tensorflow/python/ops/check_ops.py @@ -229,10 +229,14 @@ def assert_non_negative(x, data=None, summarize=None, message=None, name=None): with ops.name_scope(name, 'assert_non_negative', [x, data]): x = ops.convert_to_tensor(x, name='x') if data is None: + if context.in_eager_mode(): + name = str(x) + else: + name = x.name data = [ message, 'Condition x >= 0 did not hold element-wise:', - 'x (%s) = ' % x.name, x] + 'x (%s) = ' % name, x] zero = ops.convert_to_tensor(0, dtype=x.dtype) return assert_less_equal(zero, x, data=data, summarize=summarize) diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py index 29aac913f0135a0f715b88e25d8c6e9c24d35ddb..10d8e01304342c42a4ee20a2c9b3e4a4817d7c95 100644 --- a/tensorflow/python/ops/control_flow_ops.py +++ b/tensorflow/python/ops/control_flow_ops.py @@ -116,6 +116,7 @@ def Assert(condition, data, summarize=None, name=None): Returns: assert_op: An `Operation` that, when executed, raises a `tf.errors.InvalidArgumentError` if `condition` is not true. + @compatibility{eager} returns None. """ with ops.name_scope(name, "Assert", [condition, data]) as name: xs = ops.convert_n_to_tensor(data) @@ -132,6 +133,8 @@ def Assert(condition, data, summarize=None, name=None): condition, data, summarize, name="Assert") guarded_assert = cond( condition, no_op, true_assert, name="AssertGuard") + if context.in_eager_mode(): + return return guarded_assert.op @@ -2907,7 +2910,7 @@ def _GroupControlDeps(dev, deps, name=None): def group(*inputs, **kwargs): """Create an op that groups multiple operations. - When this op finishes, all ops in `input` have finished. This op has no + When this op finishes, all ops in `inputs` have finished. This op has no output. See also @{tf.tuple$tuple} and @@ -2915,7 +2918,6 @@ def group(*inputs, **kwargs): Args: *inputs: Zero or more tensors to group. - **kwargs: Optional parameters to pass when constructing the NodeDef. name: A name for this operation (optional). Returns: @@ -2936,7 +2938,16 @@ def group(*inputs, **kwargs): # Sorts *inputs according to their devices. ops_on_device = {} # device -> operations specified on the device. - for inp in inputs: + for inp in nest.flatten(inputs): + if not hasattr(inp, "device"): + raise TypeError("Expected tf.group() expected Tensor arguments not " + "'%s' with type '%s'" % (inp, type(inp))) + if not hasattr(inp, "device"): + if isinstance(inp, list): + raise TypeError("To call tf.group() with a list, use " + "tf.group(*[...]) not tf.group([...]).") + raise TypeError("Expected tf.group() expected Tensor arguments not " + "'%s' with type '%s'" % (inp, type(inp))) dev = inp.device if dev in ops_on_device: ops_on_device[dev].append(inp) diff --git a/tensorflow/python/ops/control_flow_ops_test.py b/tensorflow/python/ops/control_flow_ops_test.py index d4e66ff1b32674aa69d5601396d9a27c8280c312..3e8f39dd240af3a5030d259603ab648d50c27cd3 100644 --- a/tensorflow/python/ops/control_flow_ops_test.py +++ b/tensorflow/python/ops/control_flow_ops_test.py @@ -114,6 +114,23 @@ class GroupTestCase(test_util.TensorFlowTestCase): device: "/task:2" } """, self._StripGraph(gd)) + def testPassingList(self): + with ops.Graph().as_default() as g: + a = constant_op.constant(0, name="a") + b = constant_op.constant(0, name="b") + control_flow_ops.group([a.op, b.op], name="root") + gd = g.as_graph_def() + self.assertProtoEquals(""" + node { name: "a" op: "Const"} + node { name: "b" op: "Const"} + node { name: "root" op: "NoOp" input: "^a" input: "^b" } + """, self._StripGraph(gd)) + + def testPassingNonTensors(self): + with ops.Graph().as_default(): + with self.assertRaises(TypeError): + control_flow_ops.group(1, 2) + class ShapeTestCase(test_util.TensorFlowTestCase): diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py index 41dd7f1467657ff755e44fc7bf27b34cdea61fdb..c186eb5b7ecaa5c74841aca15f0f11e994eba2ea 100644 --- a/tensorflow/python/ops/data_flow_ops.py +++ b/tensorflow/python/ops/data_flow_ops.py @@ -123,6 +123,11 @@ class QueueBase(object): @{tf.RandomShuffleQueue} for concrete implementations of this class, and instructions on how to create them. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, dtypes, shapes, names, queue_ref): @@ -146,7 +151,12 @@ class QueueBase(object): Raises: ValueError: If one of the arguments is invalid. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "Queues are not supported when eager execution is enabled. " + "Instead, please use tf.data to get data into your model.") self._dtypes = dtypes if shapes is not None: if len(shapes) != len(dtypes): @@ -590,6 +600,11 @@ class RandomShuffleQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, min_after_dequeue, dtypes, shapes=None, @@ -663,6 +678,11 @@ class FIFOQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, dtypes, shapes=None, names=None, @@ -714,6 +734,11 @@ class PaddingFIFOQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, dtypes, shapes, names=None, shared_name=None, @@ -776,6 +801,11 @@ class PriorityQueue(QueueBase): See @{tf.QueueBase} for a description of the methods on this class. + + @compatibility(eager) + Queues are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, capacity, types, shapes=None, names=None, shared_name=None, diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py index 15a1125f8219dcb31c7b84fc48652a19ddb6e39a..ba25b2c3485706cc769b8f37118a994e065c1f93 100644 --- a/tensorflow/python/ops/distributions/transformed_distribution.py +++ b/tensorflow/python/ops/distributions/transformed_distribution.py @@ -503,6 +503,19 @@ class TransformedDistribution(distribution_lib.Distribution): x = self.bijector.inverse(y) return self.distribution.survival_function(x) + def _quantile(self, value): + if self._is_maybe_event_override: + raise NotImplementedError("quantile is not implemented when overriding " + "event_shape") + if not self.bijector._is_injective: # pylint: disable=protected-access + raise NotImplementedError("quantile is not implemented when " + "bijector is not injective.") + # x_q is the "qth quantile" of X iff q = P[X <= x_q]. Now, since X = + # g^{-1}(Y), q = P[X <= x_q] = P[g^{-1}(Y) <= x_q] = P[Y <= g(x_q)], + # implies the qth quantile of Y is g(x_q). + inv_cdf = self.distribution.quantile(value) + return self.bijector.forward(inv_cdf) + def _entropy(self): if not self.bijector.is_constant_jacobian: raise NotImplementedError("entropy is not implemented") diff --git a/tensorflow/python/ops/distributions/util.py b/tensorflow/python/ops/distributions/util.py index f261d996b5414c512980bd1961f28380073ce4a6..41b86f79409aef76dbd710606d09b21f34cab7ba 100644 --- a/tensorflow/python/ops/distributions/util.py +++ b/tensorflow/python/ops/distributions/util.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import check_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn @@ -1049,13 +1050,77 @@ def dimension_size(x, axis): """Returns the size of a specific dimension.""" # Since tf.gather isn't "constant-in, constant-out", we must first check the # static shape or fallback to dynamic shape. - num_rows = (None if x.get_shape().ndims is None - else x.get_shape()[axis].value) - if num_rows is not None: - return num_rows + s = x.shape.with_rank_at_least(axis + 1)[axis].value + if axis > -1 and s is not None: + return s return array_ops.shape(x)[axis] +def process_quadrature_grid_and_probs( + quadrature_grid_and_probs, dtype, validate_args, name=None): + """Validates quadrature grid, probs or computes them as necessary. + + Args: + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. When `None`, defaults to: + `np.polynomial.hermite.hermgauss(deg=8)`. + dtype: The expected `dtype` of `grid` and `probs`. + validate_args: Python `bool`, default `False`. When `True` distribution + parameters are checked for validity despite possibly degrading runtime + performance. When `False` invalid inputs may silently render incorrect + outputs. + name: Python `str` name prefixed to Ops created by this class. + + Returns: + quadrature_grid_and_probs: Python pair of `float`-like `Tensor`s + representing the sample points and the corresponding (possibly + normalized) weight. + + Raises: + ValueError: if `quadrature_grid_and_probs is not None` and + `len(quadrature_grid_and_probs[0]) != len(quadrature_grid_and_probs[1])` + """ + with ops.name_scope(name, "process_quadrature_grid_and_probs", + [quadrature_grid_and_probs]): + if quadrature_grid_and_probs is None: + grid, probs = np.polynomial.hermite.hermgauss(deg=8) + grid = grid.astype(dtype.as_numpy_dtype) + probs = probs.astype(dtype.as_numpy_dtype) + probs /= np.linalg.norm(probs, ord=1, keepdims=True) + grid = ops.convert_to_tensor(grid, name="grid", dtype=dtype) + probs = ops.convert_to_tensor(probs, name="probs", dtype=dtype) + return grid, probs + + grid, probs = tuple(quadrature_grid_and_probs) + grid = ops.convert_to_tensor(grid, name="grid", dtype=dtype) + probs = ops.convert_to_tensor(probs, name="unnormalized_probs", + dtype=dtype) + probs /= linalg_ops.norm(probs, ord=1, axis=-1, keep_dims=True, + name="probs") + + def _static_dim_size(x, axis): + """Returns the static size of a specific dimension or `None`.""" + return x.shape.with_rank_at_least(axis + 1)[axis].value + + m, n = _static_dim_size(probs, axis=0), _static_dim_size(grid, axis=0) + if m is not None and n is not None: + if m != n: + raise ValueError("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length zero-th-dimension `Tensor`s " + "(saw lengths {}, {})".format(m, n)) + elif validate_args: + grid = control_flow_ops.with_dependencies([ + check_ops.assert_equal( + dimension_size(probs, axis=0), + dimension_size(grid, axis=0), + message=("`quadrature_grid_and_probs` must be a `tuple` of " + "same-length zero-th-dimension `Tensor`s")), + ], grid) + + return grid, probs + + class AppendDocstring(object): """Helper class to promote private subclass docstring to public counterpart. diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py index 96b799f610ce051e5cf6d10b715fddf80df561bc..688512bea6b274eed2823794f017e14eb4f128f5 100644 --- a/tensorflow/python/ops/functional_ops.py +++ b/tensorflow/python/ops/functional_ops.py @@ -27,6 +27,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor @@ -87,15 +88,20 @@ def foldl(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, if not callable(fn): raise TypeError("fn must be callable.") + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "foldl", [elems]): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally + # and not issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True # Convert elems to tensor array. elems = ops.convert_to_tensor(elems, name="elems") @@ -121,7 +127,9 @@ def foldl(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, back_prop=back_prop, swap_memory=swap_memory) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return r_a @@ -167,15 +175,20 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, if not callable(fn): raise TypeError("fn must be callable.") + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "foldr", [elems]): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally and not + # issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True # Convert elems to tensor array. elems = ops.convert_to_tensor(elems, name="elems") @@ -201,7 +214,9 @@ def foldr(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, back_prop=back_prop, swap_memory=swap_memory) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return r_a @@ -324,15 +339,20 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True, elems_flat = input_flatten(elems) + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "map", elems_flat): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally + # and not issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True elems_flat = [ ops.convert_to_tensor(elem, name="elem") for elem in elems_flat] @@ -396,7 +416,9 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True, r.set_shape(tensor_shape.TensorShape(n_static).concatenate( r.get_shape()[1:])) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return output_pack(results_flat) @@ -509,15 +531,20 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, elems_flat = input_flatten(elems) + in_graph_mode = context.in_graph_mode() with ops.name_scope(name, "scan", elems_flat): - # Any get_variable calls in fn will cache the first call locally - # and not issue repeated network I/O requests for each iteration. - varscope = vs.get_variable_scope() - varscope_caching_device_was_none = False - if varscope.caching_device is None: - # TODO(ebrevdo): Change to using colocate_with here and in other methods. - varscope.set_caching_device(lambda op: op.device) - varscope_caching_device_was_none = True + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode: + # Any get_variable calls in fn will cache the first call locally + # and not issue repeated network I/O requests for each iteration. + varscope = vs.get_variable_scope() + varscope_caching_device_was_none = False + if varscope.caching_device is None: + # TODO(ebrevdo): Change to using colocate_with here and in other + # methods. + varscope.set_caching_device(lambda op: op.device) + varscope_caching_device_was_none = True # Convert elems to tensor array. elems_flat = [ @@ -594,7 +621,9 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True, r.set_shape(tensor_shape.TensorShape(n_static).concatenate( r.get_shape()[1:])) - if varscope_caching_device_was_none: + # TODO(akshayka): Remove the in_graph_mode check once caching devices are + # supported in Eager + if in_graph_mode and varscope_caching_device_was_none: varscope.set_caching_device(None) return output_pack(results_flat) diff --git a/tensorflow/python/ops/gradients_impl.py b/tensorflow/python/ops/gradients_impl.py index f7b72eb82fa36429e9a55d009483ef50d12d7a2f..97a3486f616ddcd8244f182dffeb506ee54fcdb4 100644 --- a/tensorflow/python/ops/gradients_impl.py +++ b/tensorflow/python/ops/gradients_impl.py @@ -227,53 +227,52 @@ def _DefaultGradYs(grad_ys, ys, colocate_gradients_with_ops): for i in xrange(len(grad_ys)): grad_y = grad_ys[i] y = ys[i] - if grad_y is None: - if y.dtype.is_complex: - raise TypeError( - "Gradients of complex tensors must set grad_ys (y.dtype = %r)" % - y.dtype) - with _maybe_colocate_with(y.op, colocate_gradients_with_ops): + with _maybe_colocate_with(y.op, colocate_gradients_with_ops): + if grad_y is None: + if y.dtype.is_complex: + raise TypeError( + "Gradients of complex tensors must set grad_ys (y.dtype = %r)" % + y.dtype) new_grad_ys.append(array_ops.fill( array_ops.shape(y), constant_op.constant( 1, dtype=y.dtype, name="grad_ys_%d" % i))) - continue - if y.dtype.is_floating or y.dtype.is_integer: - if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer: - raise TypeError("Gradient type %s generated for real or " - "integer-valued tensor %s with type %s must be " - "real or integer" % - (dtypes.as_dtype(grad_y.dtype).name, y, - dtypes.as_dtype(y.dtype).name)) - elif y.dtype.is_complex: - if not grad_y.dtype.is_complex: - raise TypeError("Gradient type %s generated for complex-valued " - "tensor %s with type %s must be real" % - (dtypes.as_dtype(grad_y.dtype).name, y, - dtypes.as_dtype(y.dtype).name)) - else: - raise TypeError("Tensor %s with type %s must be numeric " - "to obtain a default gradient" % - (y, dtypes.as_dtype(y.dtype).name)) - # Create a grad_y tensor in the name scope of the gradient. - # Required for TensorArrays to identify which gradient call a - # grad_y value is coming from. - if isinstance(grad_y, ops.IndexedSlices): - new_grad_ys.append( - ops.IndexedSlices( - indices=(array_ops.identity(grad_y.indices, - name="grad_ys_%d_indices" % i) - if isinstance(grad_y.indices, ops.Tensor) - else grad_y.indices), - values=(array_ops.identity(grad_y.values, - name="grad_ys_%d_values" % i) - if isinstance(grad_y.values, ops.Tensor) - else grad_y.values), - dense_shape=(array_ops.identity(grad_y.dense_shape, - name="grad_ys_%d_shape" % i) - if isinstance(grad_y.dense_shape, ops.Tensor) - else grad_y.dense_shape))) - else: - new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i)) + continue + if y.dtype.is_floating or y.dtype.is_integer: + if not grad_y.dtype.is_floating and not grad_y.dtype.is_integer: + raise TypeError("Gradient type %s generated for real or " + "integer-valued tensor %s with type %s must be " + "real or integer" % + (dtypes.as_dtype(grad_y.dtype).name, y, + dtypes.as_dtype(y.dtype).name)) + elif y.dtype.is_complex: + if not grad_y.dtype.is_complex: + raise TypeError("Gradient type %s generated for complex-valued " + "tensor %s with type %s must be real" % + (dtypes.as_dtype(grad_y.dtype).name, y, + dtypes.as_dtype(y.dtype).name)) + else: + raise TypeError("Tensor %s with type %s must be numeric " + "to obtain a default gradient" % + (y, dtypes.as_dtype(y.dtype).name)) + # Create a grad_y tensor in the name scope of the gradient. + # Required for TensorArrays to identify which gradient call a + # grad_y value is coming from. + if isinstance(grad_y, ops.IndexedSlices): + new_grad_ys.append( + ops.IndexedSlices( + indices=(array_ops.identity( + grad_y.indices, name="grad_ys_%d_indices" % i) + if isinstance(grad_y.indices, ops.Tensor) else + grad_y.indices), + values=(array_ops.identity( + grad_y.values, name="grad_ys_%d_values" % i) if isinstance( + grad_y.values, ops.Tensor) else grad_y.values), + dense_shape=(array_ops.identity( + grad_y.dense_shape, name="grad_ys_%d_shape" % i) + if isinstance(grad_y.dense_shape, ops.Tensor) else + grad_y.dense_shape))) + else: + new_grad_ys.append(array_ops.identity(grad_y, name="grad_ys_%d" % i)) return new_grad_ys @@ -583,8 +582,10 @@ def gradients(ys, # therefore dC/doutput[i] is 0. for i, out_grad in enumerate(out_grads): if (not isinstance(out_grad, ops.Tensor) and - not out_grad) and _IsTrainable(op.outputs[i]): - # Only floating-point outputs get a zero gradient. Gradient + not out_grad) and ((not grad_fn and is_func_call) or + _IsTrainable(op.outputs[i])): + # Only trainable outputs or outputs for a function call that + # will use SymbolicGradient get a zero gradient. Gradient # functions should ignore the gradient for other outputs. # TODO(apassos) gradients of resource handles might be an # issue here because of zeros. @@ -671,15 +672,15 @@ def _UpdatePendingAndEnqueueReady(grads, op, queue, pending_count, loop_state): grad_state.pending_exits_count -= 1 if grad_state.pending_exits_count == 0: # We now have all the exits so process them. - has_real_grad = False + has_not_none_grad = False for y in grad_state.deferred_exits: if _HasAnyNotNoneGrads(grads, y.op): - has_real_grad = True + has_not_none_grad = True queue.append(y.op) else: grad_state.unused_exits.append(y) - if has_real_grad: - # For an unused exit, if it has floating-point outputs, backprop + if has_not_none_grad: + # For an unused exit, if it has trainable outputs, backprop # a zero gradient. Otherwise, just ignore it. for y in grad_state.unused_exits: if _IsTrainable(y): diff --git a/tensorflow/python/ops/gradients_test.py b/tensorflow/python/ops/gradients_test.py index de3dd03486d2d22f689652e5574858ac1a04b8f1..f0cffbab3035509fed68583ceff0710ee514e109 100644 --- a/tensorflow/python/ops/gradients_test.py +++ b/tensorflow/python/ops/gradients_test.py @@ -78,6 +78,7 @@ def _OpsBetween(graph, to_ops, from_ops): return between_ops +@test_util.with_c_api class GradientsTest(test_util.TensorFlowTestCase): def _OpNames(self, op_list): @@ -264,6 +265,10 @@ class GradientsTest(test_util.TensorFlowTestCase): self.assertEqual(10.0, grads[1].eval()) def testNoGradientForStringOutputs(self): + # This test can't be run twice because the TestStringOutput gradient can + # only be registered once. Just run with the C API enabled. + if not ops._USE_C_API: return + with ops.Graph().as_default(): def _TestOpGrad(_, float_grad, string_grad): @@ -409,6 +414,7 @@ class GradientsTest(test_util.TensorFlowTestCase): np.testing.assert_allclose(a, b) +@test_util.with_c_api class FunctionGradientsTest(test_util.TensorFlowTestCase): @classmethod @@ -498,6 +504,7 @@ class FunctionGradientsTest(test_util.TensorFlowTestCase): f.add_to_graph(ops.Graph()) +@test_util.with_c_api class StopGradientTest(test_util.TensorFlowTestCase): def testStopGradient(self): @@ -508,6 +515,7 @@ class StopGradientTest(test_util.TensorFlowTestCase): assert igrad is None +@test_util.with_c_api class PreventGradientTest(test_util.TensorFlowTestCase): def testPreventGradient(self): @@ -518,6 +526,7 @@ class PreventGradientTest(test_util.TensorFlowTestCase): _ = gradients.gradients(out, inp) +@test_util.with_c_api class HessianVectorProductTest(test_util.TensorFlowTestCase): def testHessianVectorProduct(self): @@ -546,6 +555,7 @@ class HessianVectorProductTest(test_util.TensorFlowTestCase): self.assertAllClose(hess_v_value, hess_v_actual) +@test_util.with_c_api class HessianTest(test_util.TensorFlowTestCase): def testHessian1D(self): @@ -594,6 +604,7 @@ class HessianTest(test_util.TensorFlowTestCase): gradients.hessians(x, x) +@test_util.with_c_api class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): def testIndexedSlicesToTensor(self): @@ -651,6 +662,9 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): c_sparse = ops.IndexedSlices( array_ops.placeholder(dtypes.float32), array_ops.placeholder(dtypes.int32), constant([100, 100, 100, 100])) + # "always" filter prevents the warning from being suppressed if it was + # already triggered in a different test. + warnings.simplefilter("always") with warnings.catch_warnings(record=True) as w: math_ops.multiply(c_sparse, 1.0) self.assertEqual(1, len(w)) @@ -671,6 +685,7 @@ class IndexedSlicesToTensorTest(test_util.TensorFlowTestCase): str(w[0].message)) +@test_util.with_c_api class OnlyRealGradientsTest(test_util.TensorFlowTestCase): def testRealOnly(self): diff --git a/tensorflow/python/ops/hidden_ops.txt b/tensorflow/python/ops/hidden_ops.txt index 04dfb5b65d781b022778ffde084226aa4584d605..732ab8f15ab8ce7873d5454ff42cdf939cd6e5bd 100644 --- a/tensorflow/python/ops/hidden_ops.txt +++ b/tensorflow/python/ops/hidden_ops.txt @@ -244,6 +244,7 @@ TensorSummaryV2 Abs AccumulateNV2 AddN +AddV2 All Any BatchMatMul @@ -259,6 +260,7 @@ ComplexAbs Conj FloorDiv FloorMod +HistogramFixedWidth Max Mean Min diff --git a/tensorflow/python/ops/histogram_ops.py b/tensorflow/python/ops/histogram_ops.py index c2077d51af9f1d6807635ac8cef5cda80c5b2d03..51e4be9343abc6ad68786e05e9cdf87ea48e3d00 100644 --- a/tensorflow/python/ops/histogram_ops.py +++ b/tensorflow/python/ops/histogram_ops.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops +from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops @@ -69,30 +70,6 @@ def histogram_fixed_width(values, ``` """ with ops.name_scope(name, 'histogram_fixed_width', - [values, value_range, nbins]) as scope: - values = ops.convert_to_tensor(values, name='values') - values = array_ops.reshape(values, [-1]) - value_range = ops.convert_to_tensor(value_range, name='value_range') - nbins = ops.convert_to_tensor(nbins, dtype=dtypes.int32, name='nbins') - nbins_float = math_ops.cast(nbins, values.dtype) - - # Map tensor values that fall within value_range to [0, 1]. - scaled_values = math_ops.truediv(values - value_range[0], - value_range[1] - value_range[0], - name='scaled_values') - - # map tensor values within the open interval value_range to {0,.., nbins-1}, - # values outside the open interval will be zero or less, or nbins or more. - indices = math_ops.floor(nbins_float * scaled_values, name='indices') - - # Clip edge cases (e.g. value = value_range[1]) or "outliers." - indices = math_ops.cast( - clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32) - - # TODO(langmore) This creates an array of ones to add up and place in the - # bins. This is inefficient, so replace when a better Op is available. - return math_ops.unsorted_segment_sum( - array_ops.ones_like(indices, dtype=dtype), - indices, - nbins, - name=scope) + [values, value_range, nbins]) as name: + return gen_math_ops._histogram_fixed_width(values, value_range, nbins, + dtype=dtype, name=name) diff --git a/tensorflow/python/ops/histogram_ops_test.py b/tensorflow/python/ops/histogram_ops_test.py index e819e0234d2737e665757adce65897d89d56d9c6..19ad6cd2ba2b8278656a33a331995336037db356 100644 --- a/tensorflow/python/ops/histogram_ops_test.py +++ b/tensorflow/python/ops/histogram_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import numpy as np from tensorflow.python.framework import dtypes +from tensorflow.python.ops import array_ops from tensorflow.python.ops import histogram_ops from tensorflow.python.platform import test @@ -36,7 +37,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [] expected_bin_counts = [0, 0, 0, 0, 0] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) @@ -47,7 +48,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15] expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width( values, value_range, nbins=5, dtype=dtypes.int64) self.assertEqual(dtypes.int64, hist.dtype) @@ -59,7 +60,7 @@ class HistogramFixedWidthTest(test.TestCase): value_range = np.float64([0.0, 5.0]) values = np.float64([-1.0, 0.0, 1.5, 2.0, 5.0, 15]) expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) @@ -70,11 +71,29 @@ class HistogramFixedWidthTest(test.TestCase): value_range = [0.0, 5.0] values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]] expected_bin_counts = [2, 1, 1, 0, 2] - with self.test_session(): + with self.test_session(use_gpu=True): hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) self.assertEqual(dtypes.int32, hist.dtype) self.assertAllClose(expected_bin_counts, hist.eval()) + def test_shape_inference(self): + value_range = [0.0, 5.0] + values = [[-1.0, 0.0, 1.5], [2.0, 5.0, 15]] + expected_bin_counts = [2, 1, 1, 0, 2] + placeholder = array_ops.placeholder(dtypes.int32) + with self.test_session(use_gpu=True): + hist = histogram_ops.histogram_fixed_width(values, value_range, nbins=5) + self.assertAllEqual(hist.shape.as_list(), (5,)) + self.assertEqual(dtypes.int32, hist.dtype) + self.assertAllClose(expected_bin_counts, hist.eval()) + + hist = histogram_ops.histogram_fixed_width(values, value_range, + nbins=placeholder) + self.assertEquals(hist.shape.ndims, 1) + self.assertIs(hist.shape[0].value, None) + self.assertEqual(dtypes.int32, hist.dtype) + self.assertAllClose(expected_bin_counts, hist.eval({placeholder: 5})) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 4aef6ca85f3f0ae9b3c4477497ca68f3e27a1e28..2946dbe81e6d37930874689d5c95fcdbadbbc68d 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -1513,7 +1513,8 @@ def sample_distorted_bounding_box(image_size, bounding_boxes, seed=None, # Generate a single distorted bounding box. begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box( tf.shape(image), - bounding_boxes=bounding_boxes) + bounding_boxes=bounding_boxes, + min_object_covered=0.1) # Draw the bounding box in an image summary. image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0), @@ -1541,7 +1542,7 @@ def sample_distorted_bounding_box(image_size, bounding_boxes, seed=None, seed. seed2: An optional `int`. Defaults to `0`. A second seed to avoid seed collision. - min_object_covered: An optional `float`. Defaults to `0.1`. + min_object_covered: A Tensor of type `float32`. Defaults to `0.1`. The cropped area of the image must contain at least this fraction of any bounding box supplied. The value of this parameter should be non-negative. In the case of 0, the cropped area does not need to overlap diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index 348c005ff31e44e1f168d76562b0f7633be3251f..d1554b399f3776933bf970f7b2ceb8db5865d844 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -1374,6 +1374,25 @@ class PadToBoundingBoxTest(test_util.TensorFlowTestCase): y = image_ops.pad_to_bounding_box(image, 0, 0, height, width) self.assertEqual(y.get_shape().as_list(), post_shape) + def testInt64(self): + x = [1, 2, 3, + 4, 5, 6, + 7, 8, 9] + x_shape = [3, 3, 1] + + y = [0, 0, 0, + 1, 2, 3, + 4, 5, 6, + 7, 8, 9] + y_shape = [4, 3, 1] + x = np.array(x).reshape(x_shape) + y = np.array(y).reshape(y_shape) + + i = constant_op.constant([1, 0, 4, 3], dtype=dtypes.int64) + y_tf = image_ops.pad_to_bounding_box(x, i[0], i[1], i[2], i[3]) + with self.test_session(use_gpu=True): + self.assertAllClose(y, y_tf.eval()) + def testNoOp(self): x_shape = [10, 10, 10] x = np.random.uniform(size=x_shape) @@ -2434,9 +2453,13 @@ class JpegTest(test_util.TensorFlowTestCase): y, x, h, w = crop_window image1_crop = image_ops.crop_to_bounding_box(image1, y, x, h, w) - # Combined crop+decode. + # Combined decode+crop. image2 = image_ops.decode_and_crop_jpeg(jpeg0, crop_window) + # Combined decode+crop should have the same shape inference + self.assertAllEqual(image1_crop.get_shape().as_list(), + image2.get_shape().as_list()) + # CropAndDecode should be equal to DecodeJpeg+Crop. image1_crop, image2 = sess.run([image1_crop, image2]) self.assertAllEqual(image1_crop, image2) diff --git a/tensorflow/python/ops/io_ops.py b/tensorflow/python/ops/io_ops.py index bd879ac423847c07167672ee5464e146629d5eb7..670bb9a9c29e8450b101b04ce781dc97ceb78398 100644 --- a/tensorflow/python/ops/io_ops.py +++ b/tensorflow/python/ops/io_ops.py @@ -70,6 +70,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.lib.io import python_io @@ -152,6 +153,11 @@ class ReaderBase(object): contains the work units and the Reader dequeues from the queue when it is asked to produce a record (via Read()) but it has finished the last work unit. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, reader_ref, supports_serialize=False): @@ -161,7 +167,15 @@ class ReaderBase(object): reader_ref: The operation that implements the reader. supports_serialize: True if the reader implementation can serialize its state. + + Raises: + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "Readers are not supported when eager execution is enabled. " + "Instead, please use tf.data to get data into your model.") + self._reader_ref = reader_ref self._supports_serialize = supports_serialize @@ -347,6 +361,11 @@ class WholeFileReader(ReaderBase): be a filename (key) and the contents of that file (value). See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, name=None): @@ -367,6 +386,11 @@ class TextLineReader(ReaderBase): Newlines are stripped from the output. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ # TODO(josh11b): Support serializing and restoring state. @@ -390,6 +414,11 @@ class FixedLengthRecordReader(ReaderBase): """A Reader that outputs fixed-length records from a file. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ # TODO(josh11b): Support serializing and restoring state. @@ -427,6 +456,11 @@ class TFRecordReader(ReaderBase): """A Reader that outputs the records from a TFRecords file. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ # TODO(josh11b): Support serializing and restoring state. @@ -452,6 +486,11 @@ class LMDBReader(ReaderBase): """A Reader that outputs the records from a LMDB file. See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, name=None, options=None): """Create a LMDBReader. @@ -474,6 +513,11 @@ class IdentityReader(ReaderBase): work string and output (work, work). See ReaderBase for supported methods. + + @compatibility(eager) + Readers are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, name=None): diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD index b88e72a6f3710ab217391f668759e3e030ad732e..ce8c1580fe5ee614558bfd52afde0d9c5088abe6 100644 --- a/tensorflow/python/ops/linalg/BUILD +++ b/tensorflow/python/ops/linalg/BUILD @@ -9,15 +9,13 @@ py_library( srcs = glob(["*.py"]), srcs_version = "PY2AND3", deps = [ - "//tensorflow/python:array_ops", + ":linalg_impl", "//tensorflow/python:check_ops", "//tensorflow/python:control_flow_ops", "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:math_ops", "//tensorflow/python:nn", "//tensorflow/python:nn_ops", "//tensorflow/python:random_ops", - "//tensorflow/python:special_math_ops", "//tensorflow/python:tensor_util", "//tensorflow/python:util", "//third_party/py/numpy", @@ -33,6 +31,7 @@ py_library( "//tensorflow/python:array_ops", "//tensorflow/python:linalg_ops", "//tensorflow/python:math_ops", + "//tensorflow/python:special_math_ops", ], ) diff --git a/tensorflow/python/ops/linalg/linalg.py b/tensorflow/python/ops/linalg/linalg.py index 02ceb65e2a13029004ac3ab5800b62b1478a09b5..5369007a56c89ef8601f8144c2fe18717e2e78fe 100644 --- a/tensorflow/python/ops/linalg/linalg.py +++ b/tensorflow/python/ops/linalg/linalg.py @@ -18,12 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gen_linalg_ops -from tensorflow.python.ops import linalg_ops -from tensorflow.python.ops import math_ops -from tensorflow.python.ops import special_math_ops - # go/tf-wildcard-import # pylint: disable=wildcard-import,unused-import from tensorflow.python.ops.linalg.linalg_impl import * @@ -36,39 +30,15 @@ from tensorflow.python.ops.linalg.linear_operator_low_rank_update import * from tensorflow.python.ops.linalg.linear_operator_lower_triangular import * # pylint: enable=wildcard-import -# Linear algebra ops. -band_part = array_ops.matrix_band_part -cholesky = linalg_ops.cholesky -cholesky_solve = linalg_ops.cholesky_solve -det = linalg_ops.matrix_determinant -# pylint: disable=protected-access -slogdet = gen_linalg_ops._log_matrix_determinant -# pylint: disable=protected-access -diag = array_ops.matrix_diag -diag_part = array_ops.matrix_diag_part -eigh = linalg_ops.self_adjoint_eig -eigvalsh = linalg_ops.self_adjoint_eigvals -einsum = special_math_ops.einsum -eye = linalg_ops.eye -inv = linalg_ops.matrix_inverse -lstsq = linalg_ops.matrix_solve_ls -norm = linalg_ops.norm -qr = linalg_ops.qr -set_diag = array_ops.matrix_set_diag -solve = linalg_ops.matrix_solve -svd = linalg_ops.svd -tensordot = math_ops.tensordot -trace = math_ops.trace -transpose = array_ops.matrix_transpose -triangular_solve = linalg_ops.matrix_triangular_solve - # Seal API. +# pylint: disable=undefined-variable del absolute_import -del array_ops del division +del print_function +del ops +del array_ops del gen_linalg_ops del linalg_ops del math_ops -del ops -del print_function del special_math_ops +# pylint: enable=undefined-variable diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py index 1fdec2b51b5b411ae6d1971ded6321209f72ef78..04a15e3e5bc548f99bd5d4ad1fcbf0fa22b4d1ef 100644 --- a/tensorflow/python/ops/linalg/linalg_impl.py +++ b/tensorflow/python/ops/linalg/linalg_impl.py @@ -21,7 +21,35 @@ from __future__ import print_function from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_linalg_ops +from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import special_math_ops + +# Linear algebra ops. +band_part = array_ops.matrix_band_part +cholesky = linalg_ops.cholesky +cholesky_solve = linalg_ops.cholesky_solve +det = linalg_ops.matrix_determinant +# pylint: disable=protected-access +slogdet = gen_linalg_ops._log_matrix_determinant +# pylint: disable=protected-access +diag = array_ops.matrix_diag +diag_part = array_ops.matrix_diag_part +eigh = linalg_ops.self_adjoint_eig +eigvalsh = linalg_ops.self_adjoint_eigvals +einsum = special_math_ops.einsum +eye = linalg_ops.eye +inv = linalg_ops.matrix_inverse +lstsq = linalg_ops.matrix_solve_ls +norm = linalg_ops.norm +qr = linalg_ops.qr +set_diag = array_ops.matrix_set_diag +solve = linalg_ops.matrix_solve +svd = linalg_ops.svd +tensordot = math_ops.tensordot +trace = math_ops.trace +transpose = array_ops.matrix_transpose +triangular_solve = linalg_ops.matrix_triangular_solve def logdet(matrix, name=None): diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py index 0d04e29eb32c80aeabccfe0287877b5f15da30a9..27e0f17020afa0fd44ec11c49b7a77d4426933dd 100644 --- a/tensorflow/python/ops/linalg/linear_operator.py +++ b/tensorflow/python/ops/linalg/linear_operator.py @@ -693,8 +693,8 @@ class LinearOperator(object): if self._can_use_cholesky(): diag = array_ops.matrix_diag_part(self._get_cached_chol()) return 2 * math_ops.reduce_sum(math_ops.log(diag), reduction_indices=[-1]) - abs_det = math_ops.abs(self.determinant()) - return math_ops.log(abs_det) + _, log_abs_det = linalg.slogdet(self._matrix) + return log_abs_det def log_abs_determinant(self, name="log_abs_det"): """Log absolute value of determinant for every batch member. diff --git a/tensorflow/python/ops/linalg/linear_operator_test_util.py b/tensorflow/python/ops/linalg/linear_operator_test_util.py index 4a601047b6bb49c71d012469e293e07c2fa4e0b4..3d0ea3e11becae185710b140c2a84123a6b848b2 100644 --- a/tensorflow/python/ops/linalg/linear_operator_test_util.py +++ b/tensorflow/python/ops/linalg/linear_operator_test_util.py @@ -191,8 +191,7 @@ class LinearOperatorDerivedClassTest(test.TestCase): operator, mat, feed_dict = self._operator_and_mat_and_feed_dict( shape, dtype, use_placeholder=use_placeholder) op_log_abs_det = operator.log_abs_determinant() - mat_log_abs_det = math_ops.log( - math_ops.abs(linalg_ops.matrix_determinant(mat))) + _, mat_log_abs_det = linalg.slogdet(mat) if not use_placeholder: self.assertAllEqual(shape[:-2], op_log_abs_det.get_shape()) op_log_abs_det_v, mat_log_abs_det_v = sess.run( diff --git a/tensorflow/python/ops/linalg_grad.py b/tensorflow/python/ops/linalg_grad.py index ec263591e10307abf5a40e21ff6d995c10602dcb..8a76fe3ce55bbdea1677f83fe075ed3bdc8d875d 100644 --- a/tensorflow/python/ops/linalg_grad.py +++ b/tensorflow/python/ops/linalg_grad.py @@ -81,6 +81,36 @@ def _CholeskyGrad(op, grad): return grad_a * 0.5 +@ops.RegisterGradient("Qr") +def _QrGrad(op, dq, dr): + """Gradient for Qr.""" + q, r = op.outputs + if q.dtype.is_complex: + raise NotImplementedError("QrGrad not implemented for dtype: %s" % q.dtype) + if (r.shape.ndims is None or r.shape.as_list()[-2] is None or + r.shape.as_list()[-1] is None): + raise NotImplementedError("QrGrad not implemented with dynamic shapes.") + if r.shape[-2].value != r.shape[-1].value: + raise NotImplementedError("QrGrad not implemented when ncols > nrows " + "or full_matrices is true and ncols != nrows.") + + qdq = math_ops.matmul(q, dq, adjoint_a=True) + qdq_ = qdq - _linalg.adjoint(qdq) + rdr = math_ops.matmul(r, dr, adjoint_b=True) + rdr_ = rdr - _linalg.adjoint(rdr) + tril = array_ops.matrix_band_part(qdq_ + rdr_, -1, 0) + + def _TriangularSolve(x, r): + """Equiv to matmul(x, adjoint(matrix_inverse(r))) if r is upper-tri.""" + return _linalg.adjoint( + linalg_ops.matrix_triangular_solve( + r, _linalg.adjoint(x), lower=False, adjoint=False)) + + grad_a = math_ops.matmul(q, dr + _TriangularSolve(tril, r)) + grad_b = _TriangularSolve(dq - math_ops.matmul(q, qdq), r) + return grad_a + grad_b + + @ops.RegisterGradient("MatrixSolve") def _MatrixSolveGrad(op, grad): """Gradient for MatrixSolve.""" @@ -105,7 +135,7 @@ def _MatrixSolveLsGrad(op, grad): # b) Implement a symmetric rank-k update op instead of computing # x*z + transpose(x*z). This pattern occurs other places in TensorFlow. - def _overdetermined(op, grad): + def _Overdetermined(op, grad): """Gradients for the overdetermined case of MatrixSolveLs. This is the backprop for the solution to the normal equations of the first @@ -130,7 +160,7 @@ def _MatrixSolveLsGrad(op, grad): grad_b = math_ops.matmul(a, z) return (grad_a, grad_b, None) - def _underdetermined(op, grad): + def _Underdetermined(op, grad): """Gradients for the underdetermined case of MatrixSolveLs. This is the backprop for the solution to the normal equations of the second @@ -162,16 +192,16 @@ def _MatrixSolveLsGrad(op, grad): matrix_shape = op.inputs[0].get_shape()[-2:] if matrix_shape.is_fully_defined(): if matrix_shape[-2] >= matrix_shape[-1]: - return _overdetermined(op, grad) + return _Overdetermined(op, grad) else: - return _underdetermined(op, grad) + return _Underdetermined(op, grad) else: # We have to defer determining the shape to runtime and use # conditional execution of the appropriate graph. matrix_shape = array_ops.shape(op.inputs[0])[-2:] return control_flow_ops.cond(matrix_shape[-2] >= matrix_shape[-1], - lambda: _overdetermined(op, grad), - lambda: _underdetermined(op, grad)) + lambda: _Overdetermined(op, grad), + lambda: _Underdetermined(op, grad)) @ops.RegisterGradient("MatrixTriangularSolve") diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py index 1d917c22cc55bccc92828dc6d0ad4209cb935ddb..2cb467c89157b2f78c5bc3ccc037360836b00ee7 100644 --- a/tensorflow/python/ops/linalg_ops.py +++ b/tensorflow/python/ops/linalg_ops.py @@ -368,11 +368,11 @@ def self_adjoint_eigvals(tensor, name=None): def svd(tensor, full_matrices=False, compute_uv=True, name=None): - """Computes the singular value decompositions of one or more matrices. + r"""Computes the singular value decompositions of one or more matrices. Computes the SVD of each inner matrix in `tensor` such that - `tensor[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, - :])` + `tensor[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * + transpose(conj(v[..., :, :]))` ```python # a is a tensor. @@ -406,9 +406,25 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None): `[..., N, N]`. Not returned if `compute_uv` is `False`. @compatibility(numpy) - Mostly equivalent to numpy.linalg.svd, except that the order of output - arguments here is `s`, `u`, `v` when `compute_uv` is `True`, as opposed to - `u`, `s`, `v` for numpy.linalg.svd. + Mostly equivalent to numpy.linalg.svd, except that + * The order of output arguments here is `s`, `u`, `v` when `compute_uv` is + `True`, as opposed to `u`, `s`, `v` for numpy.linalg.svd. + * full_matrices is `False` by default as opposed to `True` for + numpy.linalg.svd. + * tf.linalg.svd uses the standard definition of the SVD + \\(A = U \Sigma V^H\\), such that the left singular vectors of `a` are + the columns of `u`, while the right singular vectors of `a` are the + columns of `v`. On the other hand, numpy.linalg.svd returns the adjoint + \\(V^H\\) as the third output argument. + ```python + import tensorflow as tf + import numpy as np + s, u, v = tf.linalg.svd(a) + tf_a_approx = tf.matmul(u, tf.matmul(tf.linalg.diag(s), v, adjoint_v=True)) + u, s, v_adj = np.linalg.svd(a, full_matrices=False) + np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj)) + # tf_a_approx and np_a_approx should be numerically close. + ```` @end_compatibility """ # pylint: disable=protected-access diff --git a/tensorflow/python/ops/lookup_ops.py b/tensorflow/python/ops/lookup_ops.py index 7f00344be2e71055eba98858f83db470871a73d0..fa58ffc37e212a4000bfcb56e9c8400e1e0546de 100644 --- a/tensorflow/python/ops/lookup_ops.py +++ b/tensorflow/python/ops/lookup_ops.py @@ -1011,7 +1011,7 @@ def index_table_from_tensor(vocabulary_list, Args: vocabulary_list: A 1-D `Tensor` that specifies the mapping of keys to - indices. Thetype of this object must be castable to `dtype`. + indices. The type of this object must be castable to `dtype`. num_oov_buckets: The number of out-of-vocabulary buckets. default_value: The value to use for out-of-vocabulary feature values. Defaults to -1. diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py index 3754e039ed389bf8834ce1fe8defbf536a841446..38fe093ba7236ff7fe7b580a893501c84c71f6b1 100644 --- a/tensorflow/python/ops/math_grad.py +++ b/tensorflow/python/ops/math_grad.py @@ -700,10 +700,26 @@ def _AddNGrad(op, grad): return [grad] * len(op.inputs) +def _ShapesFullySpecifiedAndEqual(x, y, grad): + # pylint: disable=protected-access + x_shape = x._shape_tuple() + y_shape = y._shape_tuple() + grad_shape = grad._shape_tuple() + # pylint: enable=protected-access + return (x_shape == y_shape and + x_shape == grad_shape and + x_shape is not None and + None not in x_shape) + + @ops.RegisterGradient("Add") def _AddGrad(op, grad): + """Gradient for Add.""" x = op.inputs[0] y = op.inputs[1] + if (isinstance(grad, ops.Tensor) and + _ShapesFullySpecifiedAndEqual(x, y, grad)): + return grad, grad sx = array_ops.shape(x) sy = array_ops.shape(y) # pylint: disable=protected-access @@ -731,10 +747,14 @@ def _MulGrad(op, grad): """The gradient of scalar multiplication.""" x = op.inputs[0] y = op.inputs[1] + # pylint: disable=protected-access + if (isinstance(grad, ops.Tensor) and + _ShapesFullySpecifiedAndEqual(x, y, grad) and + grad.dtype in (dtypes.int32, dtypes.float32)): + return gen_math_ops._mul(grad, y), gen_math_ops._mul(grad, x) assert x.dtype.base_dtype == y.dtype.base_dtype, (x.dtype, " vs. ", y.dtype) sx = array_ops.shape(x) sy = array_ops.shape(y) - # pylint: disable=protected-access rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy) # pylint: enable=protected-access x = math_ops.conj(x) diff --git a/tensorflow/python/ops/metrics.py b/tensorflow/python/ops/metrics.py index a4e2ef1dadb5be0d95fba3fb9a291468ff22217d..0465c77691c7d4e9cb80791470db8d99c64318f9 100644 --- a/tensorflow/python/ops/metrics.py +++ b/tensorflow/python/ops/metrics.py @@ -39,6 +39,7 @@ @@sensitivity_at_specificity @@sparse_average_precision_at_k @@sparse_precision_at_k +@@precision_at_top_k @@specificity_at_sensitivity @@true_negatives @@true_negatives_at_thresholds diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py index c40273b0475193322aeb306ad8a1f763ca06259e..68ec3c0101674f9641c17ad92974e1b469b458af 100644 --- a/tensorflow/python/ops/metrics_impl.py +++ b/tensorflow/python/ops/metrics_impl.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor @@ -51,10 +52,14 @@ def _local_variable(initial_value, validate_shape=True, name=None): def _remove_squeezable_dimensions(predictions, labels, weights): - """Internal version of `remove_squeezable_dimensions` which handles weights. + """Squeeze or expand last dim if needed. - Squeezes `predictions` and `labels` if their rank differs by 1. - Squeezes `weights` if its rank is 1 more than the new rank of `predictions` + Squeezes last dim of `predictions` or `labels` if their rank differs by 1 + (using confusion_matrix.remove_squeezable_dimensions). + Squeezes or expands last dim of `weights` if its rank differs by 1 from the + new rank of `predictions`. + + If `weights` is scalar, it is kept scalar. This will use static shape if available. Otherwise, it will add graph operations, which could result in a performance hit. @@ -62,12 +67,12 @@ def _remove_squeezable_dimensions(predictions, labels, weights): Args: predictions: Predicted values, a `Tensor` of arbitrary dimensions. labels: Optional label `Tensor` whose dimensions match `predictions`. - weights: Optional weight `Tensor`. It will be squeezed if its rank is 1 - more than the new rank of `predictions` + weights: Optional weight scalar or `Tensor` whose dimensions match + `predictions`. Returns: - Tuple of `predictions`, `labels` and `weights`, possibly with the last - dimension squeezed. + Tuple of `predictions`, `labels` and `weights`. Each of them possibly has + the last dimension squeezed, `weights` could be extended by one dimension. """ predictions = ops.convert_to_tensor(predictions) if labels is not None: @@ -323,7 +328,12 @@ def mean(values, weights=None, metrics_collections=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean is not supported when eager execution ' + 'is enabled.') + with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) @@ -399,7 +409,12 @@ def accuracy(labels, predictions, weights=None, metrics_collections=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.accuracy is not supported when eager ' + 'execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) @@ -626,7 +641,12 @@ def auc(labels, predictions, weights=None, num_thresholds=200, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.auc is not supported when eager execution ' + 'is enabled.') + with variable_scope.variable_scope( name, 'auc', (labels, predictions, weights)): if curve != 'ROC' and curve != 'PR': @@ -732,7 +752,12 @@ def mean_absolute_error(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_absolute_error is not supported ' + 'when eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) absolute_errors = math_ops.abs(predictions - labels) @@ -783,7 +808,12 @@ def mean_cosine_distance(labels, predictions, dim, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_cosine_distance is not supported when ' + 'eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) radial_diffs = math_ops.multiply(predictions, labels) @@ -851,7 +881,12 @@ def mean_per_class_accuracy(labels, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_per_class_accuracy is not supported ' + 'when eager execution is enabled.') + with variable_scope.variable_scope(name, 'mean_accuracy', (predictions, labels, weights)): # Check if shape is compatible. @@ -934,7 +969,12 @@ def mean_iou(labels, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_iou is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope( name, 'mean_iou', (predictions, labels, weights)): # Check if shape is compatible. @@ -1027,7 +1067,12 @@ def mean_relative_error(labels, predictions, normalizer, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_relative_error is not supported when ' + 'eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) @@ -1087,7 +1132,12 @@ def mean_squared_error(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_squared_error is not supported when ' + 'eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) squared_error = math_ops.square(labels - predictions) @@ -1136,7 +1186,12 @@ def mean_tensor(values, weights=None, metrics_collections=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.mean_tensor is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope(name, 'mean', (values, weights)): values = math_ops.to_float(values) total = _create_local('total_tensor', shape=values.get_shape()) @@ -1213,7 +1268,12 @@ def percentage_below(values, threshold, weights=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.percentage_below is not supported when ' + 'eager execution is enabled.') + is_below_threshold = math_ops.to_float(math_ops.less(values, threshold)) return mean(is_below_threshold, weights, @@ -1299,7 +1359,12 @@ def false_negatives(labels, predictions, weights=None, ValueError: If `weights` is not `None` and its shape doesn't match `values`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_negatives is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope( name, 'false_negatives', (predictions, labels, weights)): @@ -1346,7 +1411,12 @@ def false_negatives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1392,7 +1462,12 @@ def false_positives(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_positives is not supported when ' + 'eager execution is enabled.') + with variable_scope.variable_scope( name, 'false_positives', (predictions, labels, weights)): @@ -1439,7 +1514,12 @@ def false_positives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.false_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'false_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1487,7 +1567,12 @@ def true_negatives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.true_negatives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_negatives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1533,7 +1618,12 @@ def true_positives(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.true_positives is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope( name, 'true_positives', (predictions, labels, weights)): @@ -1580,7 +1670,12 @@ def true_positives_at_thresholds(labels, predictions, thresholds, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.true_positives_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'true_positives', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1639,7 +1734,12 @@ def precision(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.precision is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope( name, 'precision', (predictions, labels, weights)): @@ -1721,7 +1821,12 @@ def precision_at_thresholds(labels, predictions, thresholds, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.precision_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'precision_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -1787,7 +1892,12 @@ def recall(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.recall is not supported is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope( name, 'recall', (predictions, labels, weights)): predictions, labels, weights = _remove_squeezable_dimensions( @@ -2151,7 +2261,12 @@ def recall_at_k(labels, ValueError: If `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.recall_at_k is not ' + 'supported when eager execution is enabled.') + with ops.name_scope( name, _at_k_name('recall', k, class_id=class_id), (predictions, labels, weights)) as scope: @@ -2286,7 +2401,12 @@ def recall_at_thresholds(labels, predictions, thresholds, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.recall_at_thresholds is not ' + 'supported when eager execution is enabled.') + with variable_scope.variable_scope(name, 'recall_at_thresholds', (predictions, labels, weights)): values, update_ops = _confusion_matrix_at_thresholds( @@ -2354,7 +2474,12 @@ def root_mean_squared_error(labels, predictions, weights=None, `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.root_mean_squared_error is not ' + 'supported when eager execution is enabled.') + predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) mse, update_mse_op = mean_squared_error( @@ -2424,7 +2549,12 @@ def sensitivity_at_specificity( `weights` is not `None` and its shape doesn't match `predictions`, or if `specificity` is not between 0 and 1, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.sensitivity_at_specificity is not ' + 'supported when eager execution is enabled.') + if specificity < 0 or specificity > 1: raise ValueError('`specificity` must be in the range [0, 1].') @@ -2789,7 +2919,12 @@ def sparse_average_precision_at_k(labels, Raises: ValueError: if k is invalid. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.sparse_average_precision_at_k is not ' + 'supported when eager execution is enabled.') + if k < 1: raise ValueError('Invalid k=%s.' % k) with ops.name_scope( @@ -2901,14 +3036,14 @@ def _streaming_sparse_false_positive_at_k(labels, return var, state_ops.assign_add(var, batch_total_fp, name='update') -def _sparse_precision_at_top_k(labels, - predictions_idx, - k=None, - class_id=None, - weights=None, - metrics_collections=None, - updates_collections=None, - name=None): +def precision_at_top_k(labels, + predictions_idx, + k=None, + class_id=None, + weights=None, + metrics_collections=None, + updates_collections=None, + name=None): """Computes precision@k of the predictions with respect to sparse labels. Differs from `sparse_precision_at_k` in that predictions must be in the form @@ -2927,7 +3062,7 @@ def _sparse_precision_at_top_k(labels, N >= 1. Commonly, N=1 and predictions has shape [batch size, k]. The final dimension contains the top `k` predicted class indices. [D1, ... DN] must match `labels`. - k: Integer, k for @k metric. + k: Integer, k for @k metric. Only used for the default op name. class_id: Integer class ID for which we want binary metrics. This should be in range [0, num_classes], where num_classes is the last dimension of `predictions`. If `class_id` is outside this range, the method returns @@ -2953,9 +3088,15 @@ def _sparse_precision_at_top_k(labels, ValueError: If `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.precision_at_top_k is not ' + 'supported when eager execution is enabled.') + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), (predictions_idx, labels, weights)) as scope: + labels = _maybe_expand_labels(labels, predictions_idx) top_k_idx = math_ops.to_int64(predictions_idx) tp, tp_update = _streaming_sparse_true_positive_at_k( predictions_idx=top_k_idx, labels=labels, k=k, class_id=class_id, @@ -3047,13 +3188,16 @@ def sparse_precision_at_k(labels, ValueError: If `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.sparse_precision_at_k is not ' + 'supported when eager execution is enabled.') + with ops.name_scope(name, _at_k_name('precision', k, class_id=class_id), (predictions, labels, weights)) as scope: - labels = _maybe_expand_labels(labels, predictions) - _, top_k_idx = nn.top_k(predictions, k) - return _sparse_precision_at_top_k( + return precision_at_top_k( labels=labels, predictions_idx=top_k_idx, k=k, @@ -3115,7 +3259,12 @@ def specificity_at_sensitivity( `weights` is not `None` and its shape doesn't match `predictions`, or if `sensitivity` is not between 0 and 1, or if either `metrics_collections` or `updates_collections` are not a list or tuple. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError('tf.metrics.specificity_at_sensitivity is not ' + 'supported when eager execution is enabled.') + if sensitivity < 0 or sensitivity > 1: raise ValueError('`sensitivity` must be in the range [0, 1].') diff --git a/tensorflow/python/ops/nn.py b/tensorflow/python/ops/nn.py index a80662c8b556c2d136cf71d9833badc1371d5eb7..79af3ac11725d6c375ec379585c0f6cfe339692e 100644 --- a/tensorflow/python/ops/nn.py +++ b/tensorflow/python/ops/nn.py @@ -21,6 +21,7 @@ See the @{$python/nn} guide. @@relu @@relu6 @@crelu +@@swish @@elu @@leaky_relu @@selu diff --git a/tensorflow/python/ops/nn_grad.py b/tensorflow/python/ops/nn_grad.py index af610d8fdb29abf9542e84a92d686e433bc29556..557f39fb42e2d096b860b44e3898bb68018c0fe8 100644 --- a/tensorflow/python/ops/nn_grad.py +++ b/tensorflow/python/ops/nn_grad.py @@ -352,6 +352,13 @@ def _Relu6Grad(op, grad): return gen_nn_ops._relu6_grad(grad, op.outputs[0]) # pylint: disable=protected-access +@ops.RegisterGradient("Relu6Grad") +def _Relu6GradGrad(op, grad): + x = op.inputs[1] + return (gen_nn_ops._relu6_grad(grad, x), array_ops.zeros( + shape=array_ops.shape(x), dtype=x.dtype)) + + @ops.RegisterGradient("Elu") def _EluGrad(op, grad): return gen_nn_ops._elu_grad(grad, op.outputs[0]) @@ -934,3 +941,32 @@ def _TopKGrad(op, grad, _): validate_indices=False), in_shape), array_ops.zeros( [], dtype=dtypes.int32)] + + +@ops.RegisterGradient("NthElement") +def _NthElementGrad(op, grad): + """Return the gradients for NthElement. + + Args: + op: The NthElementOp for which we need to generate gradients. + grad: Tensor. The gradients passed to the NthElementOp + + Returns: + A list of two tensors, the first being the gradient w.r.t. the input, + the second being the gradient w.r.t. the N (None). + """ + input = op.inputs[0] + output = op.outputs[0] + + # Compute the number of elements which equal to output in each reduction + # dimension. If there are multiple elements then the gradient will be + # divided between them. + indicators = math_ops.cast( + math_ops.equal(array_ops.expand_dims(output, -1), input), + grad.dtype) + + grad = array_ops.expand_dims(grad, -1) + num_selected = array_ops.expand_dims( + math_ops.reduce_sum(indicators, -1), -1) + + return [math_ops.div(indicators, num_selected) * grad, None] diff --git a/tensorflow/python/ops/nn_grad_test.py b/tensorflow/python/ops/nn_grad_test.py new file mode 100644 index 0000000000000000000000000000000000000000..f7541c0e892819beaf27ad97d7d41b8f963a4ab9 --- /dev/null +++ b/tensorflow/python/ops/nn_grad_test.py @@ -0,0 +1,48 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for Python ops defined in nn_grad.py.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import nn_grad +from tensorflow.python.ops import nn_ops +from tensorflow.python.platform import test + + +class Relu6OpTest(test.TestCase): + def testRelu6GradGrad(self): + inputs = constant_op.constant([[-2, -1, 1, 3], [5, 7, 8, 9]], + dtype=dtypes.float32) + x_init_value = np.array([[-3.5, -1.5, 2, 4], [4.5, 7.5, 8.5, 11]]) + r = nn_ops.relu6(inputs) + r_g = gradients_impl.gradients(r, inputs)[0] + with self.test_session(): + error = gradient_checker.compute_gradient_error( + inputs, inputs.get_shape().as_list(), + r_g, r_g.get_shape().as_list(), + x_init_value=x_init_value) + self.assertLess(error, 1e-4) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/ops/nn_impl.py b/tensorflow/python/ops/nn_impl.py index db8e92831eb3be23404feadabfde8b1a9fcd770d..2c83e4e29f3875e2978f83ee47d9c9fab3909d63 100644 --- a/tensorflow/python/ops/nn_impl.py +++ b/tensorflow/python/ops/nn_impl.py @@ -22,6 +22,7 @@ import math from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import function from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import candidate_sampling_ops @@ -269,6 +270,47 @@ def relu_layer(x, weights, biases, name=None): return nn_ops.relu(xw_plus_b, name=name) +def _swish_shape(op): + """Shape helper function for swish and _swish_grad function below.""" + return [op.inputs[0].shape] + + +# Set noinline=True so that sigmoid(features) is re-computed during +# backprop, and we can free the sigmoid(features) expression immediately +# after use during the forward pass. +@function.Defun(shape_func=_swish_shape, func_name="swish_grad", noinline=True) +def _swish_grad(features, grad): + """Gradient of Swish function defined below.""" + sigmoid_features = math_ops.sigmoid(features) + activation_grad = ( + sigmoid_features * (1.0 + features * (1.0 - sigmoid_features))) + return grad * activation_grad + + +@function.Defun( + grad_func=_swish_grad, + shape_func=_swish_shape, + func_name="swish", + noinline=True) +def swish(features): + # pylint: disable=g-doc-args + """Computes the Swish activation function: `x * sigmoid(x)`. + + Source: "Swish: a Self-Gated Activation Function" (Ramachandran et al. 2017) + https://arxiv.org/abs/1710.05941 + + Args: + features: A `Tensor` representing preactivation values. + name: A name for the operation (optional). + + Returns: + The activation value. + """ + # pylint: enable=g-doc-args + features = ops.convert_to_tensor(features, name="features") + return features * math_ops.sigmoid(features) + + def l2_normalize(x, dim, epsilon=1e-12, name=None): """Normalizes along dimension `dim` using an L2 norm. diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py index 8876591e53ef7c9e44d8a25486b73665cd62b6c4..a37b68c6fa7a4b97f0e52eab7612a7b2c06fdbe0 100644 --- a/tensorflow/python/ops/nn_ops.py +++ b/tensorflow/python/ops/nn_ops.py @@ -2114,7 +2114,7 @@ def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): # pylint: di def top_k(input, k=1, sorted=True, name=None): """Finds values and indices of the `k` largest entries for the last dimension. - If the input is a vector (rank-1), finds the `k` largest entries in the vector + If the input is a vector (rank=1), finds the `k` largest entries in the vector and outputs their values and indices as vectors. Thus `values[j]` is the `j`-th largest entry in `input`, and its index is `indices[j]`. @@ -2140,6 +2140,34 @@ def top_k(input, k=1, sorted=True, name=None): return gen_nn_ops._top_kv2(input, k=k, sorted=sorted, name=name) +def nth_element(input, n, reverse=False, name=None): + r"""Finds values of the `n`-th order statistic for the last dmension. + + If the input is a vector (rank-1), finds the entries which is the nth-smallest + value in the vector and outputs their values as scalar tensor. + + For matrices (resp. higher rank input), computes the entries which is the + nth-smallest value in each row (resp. vector along the last dimension). Thus, + + values.shape = input.shape[:-1] + + Args: + input: 1-D or higher `Tensor` with last dimension at least `n+1`. + n: A `Tensor` of type `int32`. + 0-D. Position of sorted vector to select along the last dimension (along + each row for matrices). Valid range of n is `[0, input.shape[:-1])` + reverse: An optional `bool`. Defaults to `False`. + When set to True, find the nth-largest value in the vector and vice + versa. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `input`. + The `n`-th order statistic along each last dimensional slice. + """ + return gen_nn_ops.nth_element(input, n, reverse=reverse, name=name) + + def conv1d(value, filters, stride, padding, use_cudnn_on_gpu=None, data_format=None, name=None): diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py index 3528b60ca7937d7b4981a968fc6a7b646a2d18fb..3b918e4f74c64868ef74f7e26295941c6f2801ff 100644 --- a/tensorflow/python/ops/nn_test.py +++ b/tensorflow/python/ops/nn_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gradient_checker +from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_impl from tensorflow.python.ops import nn_ops from tensorflow.python.ops import partitioned_variables @@ -860,6 +861,32 @@ class LeakyReluTest(test_lib.TestCase): self.assertAllClose(outputs, [-0.2, 0.0, 0.5, 1.0, 2.0]) +class SwishTest(test_lib.TestCase): + + def testValues(self): + np_values = np.array( + [np.linspace(-10.0, 0.0, 100), + np.linspace(0.0, 10.0, 100)], + dtype=np.float32) + tf_values = constant_op.constant(np_values) + actual_tf_outputs = nn_impl.swish(tf_values) + expected_tf_outputs = tf_values * math_ops.sigmoid(tf_values) + with self.test_session() as sess: + actual_outputs, expected_outputs = sess.run( + [actual_tf_outputs, expected_tf_outputs]) + self.assertAllClose(actual_outputs, expected_outputs) + + def testGradients(self): + shape = [5, 3, 4] + sigma = 5 + input_values = np.random.randn(*shape) * sigma + x_tf = constant_op.constant(input_values) + y_tf = nn_impl.swish(x_tf) + with self.test_session(): + err = gradient_checker.compute_gradient_error(x_tf, shape, y_tf, shape) + self.assertLess(err, 1e-4) + + class MomentsTest(test_lib.TestCase): def doOutputTest(self, input_shape, moments_axes, tol=1e-4, diff --git a/tensorflow/python/ops/numerics.py b/tensorflow/python/ops/numerics.py index 4e5d4bd9a1e1039425e9f21fd3232627cc5301c6..f3558fda9ca940f2567a451bb6ad14feb10aaba7 100644 --- a/tensorflow/python/ops/numerics.py +++ b/tensorflow/python/ops/numerics.py @@ -19,6 +19,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -62,7 +63,21 @@ def add_check_numerics_ops(): Raises: ValueError: If the graph contains any numeric operations in a control flow structure. + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Not compatible with eager execution. To check for `Inf`s and `NaN`s under + eager execution, call tfe.seterr(inf_or_nan='raise') once before executing + the checked operations. + @enc_compatibility """ + if context.in_eager_mode(): + raise RuntimeError( + "add_check_numerics_ops() is not compatible with eager execution. " + "To check for Inf's and NaN's under eager execution, call " + "tfe.seterr(inf_or_nan='raise') once before executing the " + "checked operations.") + check_op = [] # This code relies on the ordering of ops in get_operations(). # The producer of a tensor always comes before that tensor's consumer in diff --git a/tensorflow/python/ops/parsing_ops.py b/tensorflow/python/ops/parsing_ops.py index ea7132791cf10ce521699233874c52aeac79c82a..14aef01dec337d7f59c799695871c8a169c3d63a 100644 --- a/tensorflow/python/ops/parsing_ops.py +++ b/tensorflow/python/ops/parsing_ops.py @@ -1183,7 +1183,7 @@ def decode_csv(records, record_defaults, field_delim=",", Each string is a record/row in the csv and all records should have the same format. record_defaults: A list of `Tensor` objects with specific types. - Acceptable types are `float32`, `int32`, `int64`, `string`. + Acceptable types are `float32`, `float64`, `int32`, `int64`, `string`. One tensor per column of the input record, with either a scalar default value for that column or empty if the column is required. field_delim: An optional `string`. Defaults to `","`. diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py index 99ff02873b6bf2fba6001958a0d6d068d0ba4ca1..9e5bb4a225e091d14936a209b82f3d250dee8359 100644 --- a/tensorflow/python/ops/resource_variable_ops.py +++ b/tensorflow/python/ops/resource_variable_ops.py @@ -26,10 +26,10 @@ from tensorflow.python.eager import tape from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_array_ops from tensorflow.python.ops import gen_resource_variable_ops +from tensorflow.python.ops import gen_state_ops from tensorflow.python.ops import variables # go/tf-wildcard-import # pylint: disable=wildcard-import @@ -43,12 +43,26 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): container = ops.get_default_graph()._container # pylint: disable=protected-access if container is None: container = "" + if not graph_mode: + # When in eager mode use a uid for the shared_name, to prevent accidental + # sharing. + shared_name = str(ops.uid()) handle = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype, shared_name=shared_name, name=name, container=container) if graph_mode: return handle + + # We do not want two distinct ResourceVariable objects for the same + # underlying resource in the runtime. + # When in eager mode, explicitly ensure so here. When in graph mode, it's + # ensured by always generating different variable names. + exists = gen_resource_variable_ops.var_is_initialized_op(handle) + if exists: + raise ValueError("variable object with name '%s' already created. Use " + "get_variable() if reuse is desired." % + shared_name) with context.graph_mode(), ops.Graph().as_default(): h = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype, shared_name=shared_name, @@ -63,6 +77,54 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode): return handle +class EagerResourceDeleter(object): + """An object which cleans up a resource handle. + + An alternative to defining a __del__ method on an object. The intended use is + that ResourceVariables or other objects with resource handles will maintain a + single reference to this object. When the parent object is collected, this + object will be too. Even if the parent object is part of a reference cycle, + the cycle will be collectable. + """ + + def __init__(self, handle, handle_device): + self._handle = handle + self._handle_device = handle_device + + def __del__(self): + # Resources follow object-identity when executing eagerly, so it is safe to + # delete the resource we have a handle to. Each Graph has a unique container + # name, which prevents resource sharing. + try: + # This resource was created in eager mode. However, this destructor may be + # running in graph mode (especially during unit tests). To clean up + # successfully, we switch back into eager mode temporarily. + with context.eager_mode(): + with ops.device(self._handle_device): + gen_resource_variable_ops.destroy_resource_op( + self._handle, ignore_lookup_error=True) + except TypeError: + # Suppress some exceptions, mainly for the case when we're running on + # module deletion. Things that can go wrong include the context module + # already being unloaded, self._handle._handle_data no longer being + # valid, and so on. Printing warnings in these cases is silly + # (exceptions raised from __del__ are printed as warnings to stderr). + pass # 'NoneType' object is not callable when the handle has been + # partially unloaded. + except AttributeError: + pass # 'NoneType' object has no attribute 'eager_mode' when context has + # been unloaded. Will catch other module unloads as well. + + +def shape_safe_assign_variable_handle(handle, shape, value, name=None): + """Helper that checks shape compatibility and assigns variable.""" + value_tensor = ops.convert_to_tensor(value) + shape.assert_is_compatible_with(value_tensor.shape) + return gen_resource_variable_ops.assign_variable_op(handle, + value_tensor, + name=name) + + class ResourceVariable(variables.Variable): """Variable based on resource handles. @@ -171,7 +233,7 @@ class ResourceVariable(variables.Variable): @compatibility(eager) When Eager Execution is enabled, the default for the `collections` argument - is None, which signifies that this Variable will not be added to any + is `None`, which signifies that this `Variable` will not be added to any collections. @end_compatibility """ @@ -248,8 +310,9 @@ class ResourceVariable(variables.Variable): @compatibility(eager) When Eager Execution is enabled, variables are never added to collections. - It is not implicitly added to the GLOBAL_VARIABLES or TRAINABLE_VARIABLES - collections, and the `collections` argument is ignored. + It is not implicitly added to the `GLOBAL_VARIABLES` or + `TRAINABLE_VARIABLES` collections, and the `collections` argument is + ignored. @end_compatibility """ if initial_value is None: @@ -270,6 +333,9 @@ class ResourceVariable(variables.Variable): collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES] self._save_slice_info = None self._in_graph_mode = context.in_graph_mode() + # Save the graph's container prefix for error checking. Reading the value of + # the ResourceVariable from another Graph in Eager mode is an error. + self._container_prefix = ops.get_default_graph()._container_prefix # pylint: disable=protected-access with ops.control_dependencies(None): with ops.name_scope(name, "Variable", [] if init_from_fn else [initial_value]) as name: @@ -296,7 +362,7 @@ class ResourceVariable(variables.Variable): self._handle_device = ( self._handle.device if self._in_graph_mode else context.get_default_context().device_name) - self._graph_shape = initial_value.get_shape() + self._shape = initial_value.get_shape() else: initial_value = initial_value() with ops.name_scope("Initializer"): @@ -311,7 +377,7 @@ class ResourceVariable(variables.Variable): self._handle_device = ( self._handle.device if self._in_graph_mode else context.get_default_context().device_name) - self._graph_shape = initial_value.get_shape() + self._shape = initial_value.get_shape() # pylint: enable=protected-access # Or get the initial value from a Tensor or Python object. @@ -336,7 +402,7 @@ class ResourceVariable(variables.Variable): graph_mode=self._in_graph_mode) self._handle_device = (self._handle.device if self._in_graph_mode else context.get_default_context().device_name) - self._graph_shape = initial_value.get_shape() + self._shape = initial_value.get_shape() self._initial_value = initial_value if self._in_graph_mode else None self._handle_name = handle_name + ":0" @@ -388,6 +454,15 @@ class ResourceVariable(variables.Variable): ops.add_to_collections(collections, self) elif ops.GraphKeys.GLOBAL_STEP in collections: ops.add_to_collections(ops.GraphKeys.GLOBAL_STEP, self) + if not self._in_graph_mode: + # After the handle has been created, set up a way to clean it up when + # executing eagerly. We'll hold the only reference to the deleter, so that + # when this object is garbage collected the deleter will be too. This + # means ResourceVariables can be part of reference cycles without those + # cycles being uncollectable, and means that no __del__ will be defined at + # all in graph mode. + self._handle_deleter = EagerResourceDeleter( + handle=self._handle, handle_device=self._handle_device) def _init_from_proto(self, variable_def, import_scope=None): """Initializes from `VariableDef` proto.""" @@ -403,7 +478,7 @@ class ResourceVariable(variables.Variable): self._handle = g.as_graph_element( ops.prepend_name_scope( variable_def.variable_name, import_scope=import_scope)) - self._graph_shape = tensor_shape.TensorShape( + self._shape = tensor_shape.TensorShape( self._handle.op.get_attr("shape")) self._handle_device = self._handle.device self._handle_name = self._handle.name @@ -427,10 +502,11 @@ class ResourceVariable(variables.Variable): self._constraint = None # LINT.ThenChange(//tensorflow/python/eager/graph_callable.py) - def __del__(self): - if context.in_eager_mode(): - gen_resource_variable_ops.destroy_resource_op(self._handle, - ignore_lookup_error=False) + def __nonzero__(self): + return self.__bool__() + + def __bool__(self): + return bool(self.read_value()) @property def dtype(self): @@ -455,11 +531,7 @@ class ResourceVariable(variables.Variable): @property def shape(self): """The shape of this variable.""" - if self._in_graph_mode: - return self._graph_shape - return tensor_shape.TensorShape( - tensor_util.constant_value( - gen_resource_variable_ops.variable_shape(self._handle))) + return self._shape @property def create(self): @@ -518,6 +590,35 @@ class ResourceVariable(variables.Variable): raise RuntimeError("Trying to eval in EAGER mode") return self._graph_element.eval(session=session) + def numpy(self): + if context.in_graph_mode(): + raise NotImplementedError( + "numpy() is only available when eager execution is enabled.") + return self.read_value().numpy() + + def count_up_to(self, limit): + """Increments this variable until it reaches `limit`. + + When that Op is run it tries to increment the variable by `1`. If + incrementing the variable would bring it above `limit` then the Op raises + the exception `OutOfRangeError`. + + If no error is raised, the Op outputs the value of the variable before + the increment. + + This is essentially a shortcut for `count_up_to(self, limit)`. + + Args: + limit: value at which incrementing the variable raises an error. + + Returns: + A `Tensor` that will hold the variable value before the increment. If no + other Op modifies this variable, the values produced will all be + distinct. + """ + return gen_state_ops.resource_count_up_to(self.handle, limit=limit, + T=self.dtype) + def _set_save_slice_info(self, save_slice_info): """Sets the slice info for this `ResourceVariable`. @@ -543,7 +644,15 @@ class ResourceVariable(variables.Variable): Returns: the read operation. + Raises: + ValueError: if the ResourceVariable was created in another isolation + environment or graph. """ + if (not self._in_graph_mode and + self._container_prefix != ops.get_default_graph()._container_prefix): # pylint: disable=protected-access + raise ValueError( + "Attempted to read a variable from another isolation environment" + " or Graph") with ops.name_scope("Read"): # Ensure we read the variable in the same device as the handle. with ops.device(self._handle_device): @@ -617,6 +726,10 @@ class ResourceVariable(variables.Variable): """Unsupported.""" raise NotImplementedError("ResourceVariable does not implement _ref()") + def set_shape(self, shape): + """Unsupported.""" + raise NotImplementedError("ResourceVariable does not implement set_shape()") + @staticmethod def _OverloadOperator(operator): # pylint: disable=invalid-name """Defer an operator overload to `ops.Tensor`. @@ -664,10 +777,12 @@ class ResourceVariable(variables.Variable): return self.read_value() def assign(self, value, use_locking=None, name=None): + value_tensor = ops.convert_to_tensor(value, dtype=self.dtype) + self._shape.assert_is_compatible_with(value_tensor.shape) with ops.control_dependencies([ gen_resource_variable_ops.assign_variable_op( self.handle, - ops.convert_to_tensor(value, dtype=self.dtype), + value_tensor, name=name) ]): return self.read_value() @@ -701,6 +816,27 @@ class ResourceVariable(variables.Variable): else: return self.value() + def __iadd__(self, unused_other): + raise RuntimeError("Variable += value not supported.") + + def __isub__(self, unused_other): + raise RuntimeError("Variable -= value not supported.") + + def __imul__(self, unused_other): + raise RuntimeError("Variable *= value not supported.") + + def __idiv__(self, unused_other): + raise RuntimeError("Variable /= value not supported.") + + def __itruediv__(self, unused_other): + raise RuntimeError("Variable /= value not supported.") + + def __irealdiv__(self, unused_other): + raise RuntimeError("Variable /= value not supported.") + + def __ipow__(self, unused_other): + raise RuntimeError("Variable **= value not supported.") + def _dense_var_to_tensor(var, dtype=None, name=None, as_ref=False): return var._dense_var_to_tensor(dtype=dtype, name=name, as_ref=as_ref) # pylint: disable=protected-access diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py index b174956e6041db918aa4b8f5a391bfbc60aa6bda..21c7ed361dc8d613d3332905ded1952dfe34681c 100644 --- a/tensorflow/python/ops/rnn.py +++ b/tensorflow/python/ops/rnn.py @@ -27,6 +27,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -576,8 +577,9 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. with vs.variable_scope(scope or "rnn") as varscope: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) + if context.in_graph_mode(): + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) batch_size = _best_effort_input_batch_size(flat_input) if initial_state is not None: @@ -595,7 +597,7 @@ def dynamic_rnn(cell, inputs, sequence_length=None, initial_state=None, ["Expected shape for Tensor %s is " % x.name, packed_shape, " but saw shape: ", x_shape]) - if sequence_length is not None: + if context.in_graph_mode() and sequence_length is not None: # Perform some shape validation with ops.control_dependencies( [_assert_has_shape(sequence_length, [batch_size])]): @@ -718,14 +720,19 @@ def _dynamic_rnn_loop(cell, size=time_steps, tensor_array_name=base_name + name) - output_ta = tuple(_create_ta("output_%d" % i, - _infer_state_dtype(dtype, state)) - for i in range(len(flat_output_size))) - input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype) - for i in range(len(flat_input))) - - input_ta = tuple(ta.unstack(input_) - for ta, input_ in zip(input_ta, flat_input)) + in_graph_mode = context.in_graph_mode() + if in_graph_mode: + output_ta = tuple(_create_ta("output_%d" % i, + _infer_state_dtype(dtype, state)) + for i in range(len(flat_output_size))) + input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype) + for i in range(len(flat_input))) + input_ta = tuple(ta.unstack(input_) + for ta, input_ in zip(input_ta, flat_input)) + else: + output_ta = tuple([0 for _ in range(time_steps.numpy())] + for i in range(len(flat_output_size))) + input_ta = flat_input def _time_step(time, output_ta_t, state): """Take a time step of the dynamic RNN. @@ -739,10 +746,13 @@ def _dynamic_rnn_loop(cell, The tuple (time + 1, output_ta_t with updated flow, new_state). """ - input_t = tuple(ta.read(time) for ta in input_ta) - # Restore some shape information - for input_, shape in zip(input_t, inputs_got_shape): - input_.set_shape(shape[1:]) + if in_graph_mode: + input_t = tuple(ta.read(time) for ta in input_ta) + # Restore some shape information + for input_, shape in zip(input_t, inputs_got_shape): + input_.set_shape(shape[1:]) + else: + input_t = tuple(ta[time.numpy()] for ta in input_ta) input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t) call_cell = lambda: cell(input_t, state) @@ -764,8 +774,12 @@ def _dynamic_rnn_loop(cell, # Pack state if using state tuples output = nest.flatten(output) - output_ta_t = tuple( - ta.write(time, out) for ta, out in zip(output_ta_t, output)) + if in_graph_mode: + output_ta_t = tuple( + ta.write(time, out) for ta, out in zip(output_ta_t, output)) + else: + for ta, out in zip(output_ta_t, output): + ta[time.numpy()] = out return (time + 1, output_ta_t, new_state) @@ -777,16 +791,20 @@ def _dynamic_rnn_loop(cell, swap_memory=swap_memory) # Unpack final output if not using output tuples. - final_outputs = tuple(ta.stack() for ta in output_final_ta) - - # Restore some shape information - for output, output_size in zip(final_outputs, flat_output_size): - shape = _concat( - [const_time_steps, const_batch_size], output_size, static=True) - output.set_shape(shape) + if in_graph_mode: + final_outputs = tuple(ta.stack() for ta in output_final_ta) + # Restore some shape information + for output, output_size in zip(final_outputs, flat_output_size): + shape = _concat( + [const_time_steps, const_batch_size], output_size, static=True) + output.set_shape(shape) + else: + final_outputs = output_final_ta final_outputs = nest.pack_sequence_as( structure=cell.output_size, flat_sequence=final_outputs) + if not in_graph_mode: + final_outputs = array_ops.stack(final_outputs, axis=0) return (final_outputs, final_state) @@ -967,8 +985,9 @@ def raw_rnn(cell, loop_fn, # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. with vs.variable_scope(scope or "rnn") as varscope: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) + if context.in_graph_mode(): + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) time = constant_op.constant(0, dtype=dtypes.int32) (elements_finished, next_input, initial_state, emit_structure, @@ -1166,8 +1185,9 @@ def static_rnn(cell, # determined by the parent scope, or is set to place the cached # Variable using the same placement as for the rest of the RNN. with vs.variable_scope(scope or "rnn") as varscope: - if varscope.caching_device is None: - varscope.set_caching_device(lambda op: op.device) + if context.in_graph_mode(): + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) # Obtain the first sequence of the input first_input = inputs diff --git a/tensorflow/python/ops/rnn_cell_impl.py b/tensorflow/python/ops/rnn_cell_impl.py index e5204b925de0634d493f1d985317710835c88320..62fd2f4d87a09527c138633d725e70d9cb542a6f 100644 --- a/tensorflow/python/ops/rnn_cell_impl.py +++ b/tensorflow/python/ops/rnn_cell_impl.py @@ -159,17 +159,17 @@ class RNNCell(base_layer.Layer): """Run this RNN cell on inputs, starting from the given state. Args: - inputs: `2-D` tensor with shape `[batch_size x input_size]`. + inputs: `2-D` tensor with shape `[batch_size, input_size]`. state: if `self.state_size` is an integer, this should be a `2-D Tensor` - with shape `[batch_size x self.state_size]`. Otherwise, if + with shape `[batch_size, self.state_size]`. Otherwise, if `self.state_size` is a tuple of integers, this should be a tuple - with shapes `[batch_size x s] for s in self.state_size`. + with shapes `[batch_size, s] for s in self.state_size`. scope: VariableScope for the created subgraph; defaults to class name. Returns: A pair containing: - - Output: A `2-D` tensor with shape `[batch_size x self.output_size]`. + - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. - New state: Either a single `2-D` tensor, or a tuple of tensors matching the arity and shapes of `state`. """ @@ -178,8 +178,13 @@ class RNNCell(base_layer.Layer): custom_getter=self._rnn_get_variable) as scope: return super(RNNCell, self).__call__(inputs, state, scope=scope) else: - with vs.variable_scope(vs.get_variable_scope(), - custom_getter=self._rnn_get_variable): + scope_attrname = "rnncell_scope" + scope = getattr(self, scope_attrname, None) + if scope is None: + scope = vs.variable_scope(vs.get_variable_scope(), + custom_getter=self._rnn_get_variable) + setattr(self, scope_attrname, scope) + with scope: return super(RNNCell, self).__call__(inputs, state) def _rnn_get_variable(self, getter, *args, **kwargs): @@ -224,15 +229,26 @@ class RNNCell(base_layer.Layer): Returns: If `state_size` is an int or TensorShape, then the return value is a - `N-D` tensor of shape `[batch_size x state_size]` filled with zeros. + `N-D` tensor of shape `[batch_size, state_size]` filled with zeros. If `state_size` is a nested list or tuple, then the return value is a nested list or tuple (of the same structure) of `2-D` tensors with - the shapes `[batch_size x s]` for each s in `state_size`. + the shapes `[batch_size, s]` for each s in `state_size`. """ + # Try to use the last cached zero_state. This is done to avoid recreating + # zeros, especially when eager execution is enabled. + state_size = self.state_size + if hasattr(self, "_last_zero_state"): + (last_state_size, last_batch_size, last_dtype, + last_output) = getattr(self, "_last_zero_state") + if (last_batch_size == batch_size and + last_dtype == dtype and + last_state_size == state_size): + return last_output with ops.name_scope(type(self).__name__ + "ZeroState", values=[batch_size]): - state_size = self.state_size - return _zero_state_tensors(state_size, batch_size, dtype) + output = _zero_state_tensors(state_size, batch_size, dtype) + self._last_zero_state = (state_size, batch_size, dtype, output) + return output class BasicRNNCell(RNNCell): @@ -333,6 +349,45 @@ class SRUCell(RNNCell): return h, c +class _LayerRNNCell(RNNCell): + """Subclass of RNNCells that act like proper `tf.Layer` objects. + + For backwards compatibility purposes, most `RNNCell` instances allow their + `call` methods to instantiate variables via `tf.get_variable`. The underlying + variable scope thus keeps track of any variables, and returning cached + versions. This is atypical of `tf.layer` objects, which separate this + part of layer building into a `build` method that is only called once. + + Here we provide a subclass for `RNNCell` objects that act exactly as + `Layer` objects do. They must provide a `build` method and their + `call` methods do not access Variables `tf.get_variable`. + """ + + def __call__(self, inputs, state, scope=None): + """Run this RNN cell on inputs, starting from the given state. + + Args: + inputs: `2-D` tensor with shape `[batch_size, input_size]`. + state: if `self.state_size` is an integer, this should be a `2-D Tensor` + with shape `[batch_size, self.state_size]`. Otherwise, if + `self.state_size` is a tuple of integers, this should be a tuple + with shapes `[batch_size, s] for s in self.state_size`. + scope: `VariableScope` for the created subgraph; if not provided, + defaults to standard `tf.layers.Layer` behavior. + + Returns: + A pair containing: + + - Output: A `2-D` tensor with shape `[batch_size, self.output_size]`. + - New state: Either a single `2-D` tensor, or a tuple of tensors matching + the arity and shapes of `state`. + """ + # Bypass RNNCell's variable capturing semantics for LayerRNNCell. + # Instead, it is up to subclasses to provide a proper build + # method. See the class docstring for more details. + return base_layer.Layer.__call__(self, inputs, state, scope=scope) + + class GRUCell(RNNCell): """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). @@ -422,7 +477,7 @@ class LSTMStateTuple(_LSTMStateTuple): return c.dtype -class BasicLSTMCell(RNNCell): +class BasicLSTMCell(_LayerRNNCell): """Basic LSTM recurrent network cell. The implementation is based on: http://arxiv.org/abs/1409.2329. @@ -438,7 +493,7 @@ class BasicLSTMCell(RNNCell): """ def __init__(self, num_units, forget_bias=1.0, - state_is_tuple=True, activation=None, reuse=None): + state_is_tuple=True, activation=None, reuse=None, name=None): """Initialize the basic LSTM cell. Args: @@ -453,14 +508,21 @@ class BasicLSTMCell(RNNCell): reuse: (optional) Python boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. + name: String, the name of the layer. Layers with the same name will + share weights, but to avoid mistakes we require reuse=True in such + cases. When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. + `CudnnCompatibleLSTMCell` instead. """ - super(BasicLSTMCell, self).__init__(_reuse=reuse) + super(BasicLSTMCell, self).__init__(_reuse=reuse, name=name) if not state_is_tuple: logging.warn("%s: Using a concatenated state is slower and will soon be " "deprecated. Use state_is_tuple=True.", self) + + # Inputs must be 2-dimensional. + self.input_spec = base_layer.InputSpec(ndim=2) + self._num_units = num_units self._forget_bias = forget_bias self._state_is_tuple = state_is_tuple @@ -476,15 +538,32 @@ class BasicLSTMCell(RNNCell): def output_size(self): return self._num_units + def build(self, inputs_shape): + if inputs_shape[1].value is None: + raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" + % inputs_shape) + + input_depth = inputs_shape[1].value + h_depth = self._num_units + self._kernel = self.add_variable( + _WEIGHTS_VARIABLE_NAME, + shape=[input_depth + h_depth, 4 * self._num_units]) + self._bias = self.add_variable( + _BIAS_VARIABLE_NAME, + shape=[4 * self._num_units], + initializer=init_ops.constant_initializer(0.0, dtype=self.dtype)) + + self._built = True + def call(self, inputs, state): """Long short-term memory cell (LSTM). Args: - inputs: `2-D` tensor with shape `[batch_size x input_size]`. + inputs: `2-D` tensor with shape `[batch_size, input_size]`. state: An `LSTMStateTuple` of state tensors, each shaped - `[batch_size x self.state_size]`, if `state_is_tuple` has been set to + `[batch_size, self.state_size]`, if `state_is_tuple` has been set to `True`. Otherwise, a `Tensor` shaped - `[batch_size x 2 * self.state_size]`. + `[batch_size, 2 * self.state_size]`. Returns: A pair containing the new hidden state, and the new state (either a @@ -492,21 +571,29 @@ class BasicLSTMCell(RNNCell): `state_is_tuple`). """ sigmoid = math_ops.sigmoid + one = constant_op.constant(1, dtype=dtypes.int32) # Parameters of gates are concatenated into one multiply for efficiency. if self._state_is_tuple: c, h = state else: - c, h = array_ops.split(value=state, num_or_size_splits=2, axis=1) + c, h = array_ops.split(value=state, num_or_size_splits=2, axis=one) + + gate_inputs = math_ops.matmul( + array_ops.concat([inputs, h], 1), self._kernel) + gate_inputs = nn_ops.bias_add(gate_inputs, self._bias) - if self._linear is None: - self._linear = _Linear([inputs, h], 4 * self._num_units, True) # i = input_gate, j = new_input, f = forget_gate, o = output_gate i, j, f, o = array_ops.split( - value=self._linear([inputs, h]), num_or_size_splits=4, axis=1) + value=gate_inputs, num_or_size_splits=4, axis=one) - new_c = ( - c * sigmoid(f + self._forget_bias) + sigmoid(i) * self._activation(j)) - new_h = self._activation(new_c) * sigmoid(o) + forget_bias_tensor = constant_op.constant(self._forget_bias, dtype=f.dtype) + # Note that using `add` and `multiply` instead of `+` and `*` gives a + # performance improvement. So using those at the cost of readability. + add = math_ops.add + multiply = math_ops.multiply + new_c = add(multiply(c, sigmoid(add(f, forget_bias_tensor))), + multiply(sigmoid(i), self._activation(j))) + new_h = multiply(self._activation(new_c), sigmoid(o)) if self._state_is_tuple: new_state = LSTMStateTuple(new_c, new_h) @@ -515,7 +602,7 @@ class BasicLSTMCell(RNNCell): return new_h, new_state -class LSTMCell(RNNCell): +class LSTMCell(_LayerRNNCell): """Long short-term memory unit (LSTM) recurrent network cell. The default non-peephole implementation is based on: @@ -542,7 +629,7 @@ class LSTMCell(RNNCell): initializer=None, num_proj=None, proj_clip=None, num_unit_shards=None, num_proj_shards=None, forget_bias=1.0, state_is_tuple=True, - activation=None, reuse=None): + activation=None, reuse=None, name=None): """Initialize the parameters for an LSTM cell. Args: @@ -572,11 +659,14 @@ class LSTMCell(RNNCell): reuse: (optional) Python boolean describing whether to reuse variables in an existing scope. If not `True`, and the existing scope already has the given variables, an error is raised. + name: String, the name of the layer. Layers with the same name will + share weights, but to avoid mistakes we require reuse=True in such + cases. - When restoring from CudnnLSTM-trained checkpoints, must use - CudnnCompatibleLSTMCell instead. + When restoring from CudnnLSTM-trained checkpoints, use + `CudnnCompatibleLSTMCell` instead. """ - super(LSTMCell, self).__init__(_reuse=reuse) + super(LSTMCell, self).__init__(_reuse=reuse, name=name) if not state_is_tuple: logging.warn("%s: Using a concatenated state is slower and will soon be " "deprecated. Use state_is_tuple=True.", self) @@ -586,6 +676,9 @@ class LSTMCell(RNNCell): "deprecated and will be removed in Jan 2017. " "Use a variable scope with a partitioner instead.", self) + # Inputs must be 2-dimensional. + self.input_spec = base_layer.InputSpec(ndim=2) + self._num_units = num_units self._use_peepholes = use_peepholes self._cell_clip = cell_clip @@ -608,12 +701,6 @@ class LSTMCell(RNNCell): LSTMStateTuple(num_units, num_units) if state_is_tuple else 2 * num_units) self._output_size = num_units - self._linear1 = None - self._linear2 = None - if self._use_peepholes: - self._w_f_diag = None - self._w_i_diag = None - self._w_o_diag = None @property def state_size(self): @@ -623,20 +710,61 @@ class LSTMCell(RNNCell): def output_size(self): return self._output_size + def build(self, inputs_shape): + if inputs_shape[1].value is None: + raise ValueError("Expected inputs.shape[-1] to be known, saw shape: %s" + % inputs_shape) + + input_depth = inputs_shape[1].value + h_depth = self._num_units if self._num_proj is None else self._num_proj + maybe_partitioner = ( + partitioned_variables.fixed_size_partitioner(self._num_unit_shards) + if self._num_unit_shards is not None + else None) + self._kernel = self.add_variable( + _WEIGHTS_VARIABLE_NAME, + shape=[input_depth + h_depth, 4 * self._num_units], + initializer=self._initializer, + partitioner=maybe_partitioner) + self._bias = self.add_variable( + _BIAS_VARIABLE_NAME, + shape=[4 * self._num_units], + initializer=init_ops.constant_initializer(0.0, dtype=self.dtype)) + if self._use_peepholes: + self._w_f_diag = self.add_variable("w_f_diag", shape=[self._num_units], + initializer=self._initializer) + self._w_i_diag = self.add_variable("w_i_diag", shape=[self._num_units], + initializer=self._initializer) + self._w_o_diag = self.add_variable("w_o_diag", shape=[self._num_units], + initializer=self._initializer) + + if self._num_proj is not None: + maybe_proj_partitioner = ( + partitioned_variables.fixed_size_partitioner(self._num_proj_shards) + if self._num_proj_shards is not None + else None) + self._proj_kernel = self.add_variable( + "projection/%s" % _WEIGHTS_VARIABLE_NAME, + shape=[self._num_units, self._num_proj], + initializer=self._initializer, + partitioner=maybe_proj_partitioner) + + self._built = True + def call(self, inputs, state): """Run one step of LSTM. Args: - inputs: input Tensor, 2D, batch x num_units. + inputs: input Tensor, 2D, `[batch, num_units]. state: if `state_is_tuple` is False, this must be a state Tensor, - `2-D, batch x state_size`. If `state_is_tuple` is True, this must be a + `2-D, [batch, state_size]`. If `state_is_tuple` is True, this must be a tuple of state Tensors, both `2-D`, with column sizes `c_state` and `m_state`. Returns: A tuple containing: - - A `2-D, [batch x output_dim]`, Tensor representing the output of the + - A `2-D, [batch, output_dim]`, Tensor representing the output of the LSTM after reading `inputs` when previous state was `state`. Here output_dim is: num_proj if num_proj was set, @@ -657,37 +785,18 @@ class LSTMCell(RNNCell): c_prev = array_ops.slice(state, [0, 0], [-1, self._num_units]) m_prev = array_ops.slice(state, [0, self._num_units], [-1, num_proj]) - dtype = inputs.dtype input_size = inputs.get_shape().with_rank(2)[1] if input_size.value is None: raise ValueError("Could not infer input size from inputs.get_shape()[-1]") - if self._linear1 is None: - scope = vs.get_variable_scope() - with vs.variable_scope( - scope, initializer=self._initializer) as unit_scope: - if self._num_unit_shards is not None: - unit_scope.set_partitioner( - partitioned_variables.fixed_size_partitioner( - self._num_unit_shards)) - self._linear1 = _Linear([inputs, m_prev], 4 * self._num_units, True) # i = input_gate, j = new_input, f = forget_gate, o = output_gate - lstm_matrix = self._linear1([inputs, m_prev]) + lstm_matrix = math_ops.matmul( + array_ops.concat([inputs, m_prev], 1), self._kernel) + lstm_matrix = nn_ops.bias_add(lstm_matrix, self._bias) + i, j, f, o = array_ops.split( value=lstm_matrix, num_or_size_splits=4, axis=1) # Diagonal connections - if self._use_peepholes and not self._w_f_diag: - scope = vs.get_variable_scope() - with vs.variable_scope( - scope, initializer=self._initializer) as unit_scope: - with vs.variable_scope(unit_scope): - self._w_f_diag = vs.get_variable( - "w_f_diag", shape=[self._num_units], dtype=dtype) - self._w_i_diag = vs.get_variable( - "w_i_diag", shape=[self._num_units], dtype=dtype) - self._w_o_diag = vs.get_variable( - "w_o_diag", shape=[self._num_units], dtype=dtype) - if self._use_peepholes: c = (sigmoid(f + self._forget_bias + self._w_f_diag * c_prev) * c_prev + sigmoid(i + self._w_i_diag * c_prev) * self._activation(j)) @@ -705,16 +814,7 @@ class LSTMCell(RNNCell): m = sigmoid(o) * self._activation(c) if self._num_proj is not None: - if self._linear2 is None: - scope = vs.get_variable_scope() - with vs.variable_scope(scope, initializer=self._initializer): - with vs.variable_scope("projection") as proj_scope: - if self._num_proj_shards is not None: - proj_scope.set_partitioner( - partitioned_variables.fixed_size_partitioner( - self._num_proj_shards)) - self._linear2 = _Linear(m, self._num_proj, False) - m = self._linear2(m) + m = math_ops.matmul(m, self._proj_kernel) if self._proj_clip is not None: # pylint: disable=invalid-unary-operand-type @@ -1185,7 +1285,7 @@ class _Linear(object): """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. + args: a 2D Tensor or a list of 2D, batch, n, Tensors. output_size: int, second dimension of weight variable. dtype: data type for variables. build_bias: boolean, whether to build a bias variable. @@ -1250,7 +1350,9 @@ class _Linear(object): if len(args) == 1: res = math_ops.matmul(args[0], self._weights) else: - res = math_ops.matmul(array_ops.concat(args, 1), self._weights) + # Explicitly creating a one for a minor performance improvement. + one = constant_op.constant(1, dtype=dtypes.int32) + res = math_ops.matmul(array_ops.concat(args, one), self._weights) if self._build_bias: res = nn_ops.bias_add(res, self._biases) return res @@ -1265,7 +1367,7 @@ def _linear(args, """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. Args: - args: a 2D Tensor or a list of 2D, batch x n, Tensors. + args: a 2D Tensor or a list of 2D, batch, n, Tensors. output_size: int, second dimension of W[i]. bias: boolean, whether to add a bias term or not. bias_initializer: starting value to initialize the bias @@ -1273,7 +1375,7 @@ def _linear(args, kernel_initializer: starting value to initialize the weight. Returns: - A 2D Tensor with shape [batch x output_size] equal to + A 2D Tensor with shape `[batch, output_size]` equal to sum_i(args[i] * W[i]), where W[i]s are newly created matrices. Raises: diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py index 87561cff92a3e7a990aeb207ebc4517bc5a285ed..fe3f7343222f7b10bc6af272146e8960d6f39c3d 100644 --- a/tensorflow/python/ops/special_math_ops.py +++ b/tensorflow/python/ops/special_math_ops.py @@ -251,9 +251,13 @@ def _einsum_reduction(t0, t0_axis_labels, t1, t1_axis_labels, axes_to_sum): `t1_axis_labels`. """ if len(t0_axis_labels) != len(t0.get_shape()): - raise ValueError() + raise ValueError( + 'Tensor t0 of rank %d does not match einsum reduction of length %d' % + (len(t0.get_shape()), len(t0_axis_labels))) if len(t1_axis_labels) != len(t1.get_shape()): - raise ValueError() + raise ValueError( + 'Tensor t1 of rank %d does not match einsum reduction of length %d' % + (len(t1.get_shape()), len(t1_axis_labels))) # This function computes the result of a two-argument einsum() using batch # matrix multiplication. This involves diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py index 65ec2d4b772782a11014ebdb205767834097ef3e..dbab07da42671744284d703f0cd80e601a5fa8a8 100644 --- a/tensorflow/python/ops/state_ops.py +++ b/tensorflow/python/ops/state_ops.py @@ -275,3 +275,77 @@ def assign(ref, value, validate_shape=None, use_locking=None, name=None): ref, value, use_locking=use_locking, name=name, validate_shape=validate_shape) return ref.assign(value) + + +def count_up_to(ref, limit, name=None): + r"""Increments 'ref' until it reaches 'limit'. + + Args: + ref: A Variable. Must be one of the following types: `int32`, `int64`. + Should be from a scalar `Variable` node. + limit: An `int`. + If incrementing ref would bring it above limit, instead generates an + 'OutOfRange' error. + name: A name for the operation (optional). + + Returns: + A `Tensor`. Has the same type as `ref`. + A copy of the input before increment. If nothing else modifies the + input, the values produced will all be distinct. + """ + if ref.dtype._is_ref_dtype: + return gen_state_ops.count_up_to(ref, limit=limit, name=name) + return gen_state_ops.resource_count_up_to( + ref.handle, limit, T=ref.dtype, name=name) + + +def scatter_update(ref, indices, updates, use_locking=True, name=None): + # pylint: disable=line-too-long + r"""Applies sparse updates to a variable reference. + + This operation computes + + ```python + # Scalar indices + ref[indices, ...] = updates[...] + + # Vector indices (for each i) + ref[indices[i], ...] = updates[i, ...] + + # High rank indices (for each i, ..., j) + ref[indices[i, ..., j], ...] = updates[i, ..., j, ...] + ``` + + This operation outputs `ref` after the update is done. + This makes it easier to chain operations that need to use the reset value. + + If values in `ref` is to be updated more than once, because there are + duplicate entries in `indices`, the order at which the updates happen + for each value is undefined. + + Requires `updates.shape = indices.shape + ref.shape[1:]`. + +
+ +
+ + Args: + ref: A `Variable`. + indices: A `Tensor`. Must be one of the following types: `int32`, `int64`. + A tensor of indices into the first dimension of `ref`. + updates: A `Tensor`. Must have the same type as `ref`. + A tensor of updated values to store in `ref`. + use_locking: An optional `bool`. Defaults to `True`. + If True, the assignment will be protected by a lock; + otherwise the behavior is undefined, but may exhibit less contention. + name: A name for the operation (optional). + + Returns: + Same as `ref`. Returned as a convenience for operations that want + to use the updated values after the update is done. + """ + if ref.dtype._is_ref_dtype: + return gen_state_ops.scatter_update(ref, indices, updates, + use_locking=use_locking, name=name) + return gen_resource_variable_ops.resource_scatter_update( + ref.handle, indices, updates, name=name) diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py index fab808a1679e7c5af3a9ee4739d2a5ac4e997bd8..24ef70c6f4d29e752ffd6ead08952fd53f5ca581 100644 --- a/tensorflow/python/ops/template.py +++ b/tensorflow/python/ops/template.py @@ -279,6 +279,16 @@ class Template(object): self._variables_created = True return result + @property + def name(self): + """Returns the name given to this Template.""" + return self._name + + @property + def func(self): + """Returns the func given to this Template.""" + return self._func + @property def variable_scope(self): """Returns the variable scope object created by this Template.""" diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index 37b4b3bcf90ad2c38a371a43f559283738e40d5b..b4b7ad9d9104a2168b61ad6c3062e125be507747 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -25,6 +25,9 @@ from __future__ import print_function import contextlib from tensorflow.python.eager import context +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_util @@ -34,15 +37,11 @@ from tensorflow.python.ops import math_ops from tensorflow.python.util import tf_should_use -# TensorArray object accesses many of the hidden generated ops, but is -# in fact built to wrap these methods. +# _GraphTensorArray accesses many of the hidden generated ops, but is in +# fact built to wrap these methods. # pylint: disable=protected-access -class TensorArray(object): - """Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays. - - This class is meant to be used with dynamic iteration primitives such as - `while_loop` and `map_fn`. It supports gradient back-propagation via special - "flow" control flow dependencies. +class _GraphTensorArray(object): + """Graph-mode implementation of TensorArray. """ def __init__(self, @@ -57,14 +56,7 @@ class TensorArray(object): element_shape=None, colocate_with_first_write_call=True, name=None): - """Construct a new TensorArray or wrap an existing TensorArray handle. - - A note about the parameter `name`: - - The name of the `TensorArray` (even if passed in) is uniquified: each time - a new `TensorArray` is created at runtime it is assigned its own name for - the duration of the run. This avoids name collisions if a `TensorArray` - is created within a `while_loop`. + """Constructs a graph mode TensorArray. Args: dtype: (required) data type of the TensorArray. @@ -79,9 +71,9 @@ class TensorArray(object): This is used when creating the TensorArray handle. If this value is set, handle should be None. handle: (optional) A `Tensor` handle to an existing TensorArray. If this - is set, tensor_array_name should be None. + is set, tensor_array_name should be None. Only supported in graph mode. flow: (optional) A float `Tensor` scalar coming from an existing - `TensorArray.flow`. + `TensorArray.flow`. Only supported in graph mode. infer_shape: (optional, default: True) If True, shape inference is enabled. In this case, all elements must have the same shape. element_shape: (optional, default: None) A `TensorShape` object specifying @@ -170,17 +162,14 @@ class TensorArray(object): @property def flow(self): - """The flow `Tensor` forcing ops leading to this TensorArray state.""" return self._flow @property def dtype(self): - """The data type of this TensorArray.""" return self._dtype @property def handle(self): - """The reference to the TensorArray.""" return self._handle def _merge_element_shape(self, shape): @@ -225,13 +214,7 @@ class TensorArray(object): yield def identity(self): - """Returns a TensorArray with the same content and properties. - - Returns: - A new TensorArray object with flow that ensures the control dependencies - from the contexts will become control dependencies for writes, reads, etc. - Use this object all for subsequent operations. - """ + """See TensorArray.""" flow = array_ops.identity(self._flow) ta = TensorArray( dtype=self._dtype, handle=self._handle, flow=flow, @@ -242,6 +225,7 @@ class TensorArray(object): return ta def grad(self, source, flow=None, name=None): + """See TensorArray.""" # tensor_array_grad requires a flow input when forward # TensorArrays are dynamically sized. This forces the creation # of the grad TensorArray only once the final forward array's size @@ -264,15 +248,7 @@ class TensorArray(object): return g def read(self, index, name=None): - """Read the value at location `index` in the TensorArray. - - Args: - index: 0-D. int32 tensor with the index to read from. - name: A name for the operation (optional). - - Returns: - The tensor at index `index`. - """ + """See TensorArray.""" value = gen_data_flow_ops._tensor_array_read_v3( handle=self._handle, index=index, @@ -285,20 +261,7 @@ class TensorArray(object): @tf_should_use.should_use_result def write(self, index, value, name=None): - """Write `value` into index `index` of the TensorArray. - - Args: - index: 0-D. int32 scalar with the index to write to. - value: N-D. Tensor of type `dtype`. The Tensor to write to this index. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the write occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if there are more writers than specified. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArrayWrite", [self._handle, index, value]): value = ops.convert_to_tensor(value, name="value") if self._infer_shape: @@ -319,35 +282,13 @@ class TensorArray(object): return ta def stack(self, name=None): - """Return the values in the TensorArray as a stacked `Tensor`. - - All of the values must have been written and their shapes must all match. - If input shapes have rank-`R`, then output shape will have rank-`(R+1)`. - - Args: - name: A name for the operation (optional). - - Returns: - All the tensors in the TensorArray stacked into one tensor. - """ + """See TensorArray.""" with ops.colocate_with(self._handle): with ops.name_scope(name, "TensorArrayStack", [self._handle]): return self.gather(math_ops.range(0, self.size()), name=name) def gather(self, indices, name=None): - """Return selected values in the TensorArray as a packed `Tensor`. - - All of selected values must have been written and their shapes - must all match. - - Args: - indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If - the `TensorArray` is not dynamic, `max_value=size()`. - name: A name for the operation (optional). - - Returns: - The in the `TensorArray` selected by `indices`, packed into one tensor. - """ + """See TensorArray.""" if self._element_shape: element_shape = self._element_shape[0] else: @@ -364,17 +305,7 @@ class TensorArray(object): return value def concat(self, name=None): - """Return the values in the TensorArray as a concatenated `Tensor`. - - All of the values must have been written, their ranks must match, and - and their shapes must all match for all dimensions except the first. - - Args: - name: A name for the operation (optional). - - Returns: - All the tensors in the TensorArray concatenated into one tensor. - """ + """See TensorArray.""" if self._element_shape and self._element_shape[0].dims is not None: element_shape_except0 = ( tensor_shape.TensorShape(self._element_shape[0].dims[1:])) @@ -392,22 +323,7 @@ class TensorArray(object): @tf_should_use.should_use_result def unstack(self, value, name=None): - """Unstack the values of a `Tensor` in the TensorArray. - - If input value shapes have rank-`R`, then the output TensorArray will - contain elements whose shapes are rank-`(R-1)`. - - Args: - value: (N+1)-D. Tensor of type `dtype`. The Tensor to unstack. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the unstack occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if the shape inference fails. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArrayUnstack", [self._handle, value]): num_elements = array_ops.shape(value)[0] return self.scatter( @@ -415,21 +331,7 @@ class TensorArray(object): @tf_should_use.should_use_result def scatter(self, indices, value, name=None): - """Scatter the values of a `Tensor` in specific indices of a `TensorArray`. - - Args: - indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If - the `TensorArray` is not dynamic, `max_value=size()`. - value: (N+1)-D. Tensor of type `dtype`. The Tensor to unpack. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the scatter occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if the shape inference fails. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArrayScatter", [self._handle, value, indices]): value = ops.convert_to_tensor(value, name="value") @@ -452,21 +354,7 @@ class TensorArray(object): @tf_should_use.should_use_result def split(self, value, lengths, name=None): - """Split the values of a `Tensor` into the TensorArray. - - Args: - value: (N+1)-D. Tensor of type `dtype`. The Tensor to split. - lengths: 1-D. int32 vector with the lengths to use when splitting - `value` along its first dimension. - name: A name for the operation (optional). - - Returns: - A new TensorArray object with flow that ensures the split occurs. - Use this object all for subsequent operations. - - Raises: - ValueError: if the shape inference fails. - """ + """See TensorArray.""" with ops.name_scope(name, "TensorArraySplit", [self._handle, value, lengths]): value = ops.convert_to_tensor(value, name="value") @@ -494,14 +382,627 @@ class TensorArray(object): return ta def size(self, name=None): - """Return the size of the TensorArray.""" + """See TensorArray.""" return gen_data_flow_ops._tensor_array_size_v3( handle=self._handle, flow_in=self.flow, name=name) @tf_should_use.should_use_result def close(self, name=None): - """Close the current TensorArray.""" + """See TensorArray.""" return gen_data_flow_ops._tensor_array_close_v3( handle=self._handle, name=name) # pylint: enable=protected-access + + +# pylint: disable=protected-access +def _eager_write_no_copy(ta, index, value): + """Writes value into an _EagerTensorArray without creating a new TensorArray. + + Args: + ta: _EagerTensorArray into which to write value. + index: 0-D. int32 scalar with the index to write to. + value: N-D. Tensor of type `dtype`. The Tensor to write to this index. + + Raises: + errors_impl.AlreadyExistsError: attempting to overwrite an entry. + errors_impl.InvalidArgumentError: value dtype does not match `ta`'s dtype. + errors_impl.OutOfRangeError: `index` is out of bounds. + ValueError: shape of `value` is not consistent with inferred shape. + """ + + if isinstance(index, ops.EagerTensor): + index = index.numpy() + + if index < 0: + raise errors_impl.OutOfRangeError( + None, None, + "Writing to negative indices (index %d) is not allowed." % index) + + tensor_array = ta._tensor_array + size = len(tensor_array) + if index >= size: + if not ta._dynamic_size: + raise errors_impl.OutOfRangeError( + None, None, + "Tried to write to index %d but array is not resizeable and size " + "is: %d" % (index, size)) + tensor_array.extend([None for _ in range(index - size + 1)]) + + if not isinstance(value, ops.EagerTensor): + value = constant_op.constant(value) + + if ta._infer_shape: + if ta._element_shape is None: + ta._element_shape = value.shape + elif ta._element_shape != value.shape: + raise ValueError("Incompatible shape for value (%s), expected (%s)" % + (value.shape.as_list(), ta._element_shape.as_list())) + + if ta._dtype != value.dtype: + raise errors_impl.InvalidArgumentError( + None, None, + "TensorArray dtype is %s but Op is trying to write dtype %s" % + (ta._dtype.name, value.dtype.name)) + + if ta._tensor_array[index] is not None: + raise errors_impl.AlreadyExistsError( + None, None, + "Could not write to TensorArray index %d because it has already been " + "written to." % index) + + tensor_array[index] = value + +# pylint: enable=protected-access + + +class _EagerTensorArray(object): + """Eager-mode implementation of TensorArray. + """ + + def __init__(self, + dtype, + size=None, + dynamic_size=None, + clear_after_read=None, + tensor_array_name=None, + handle=None, + flow=None, + infer_shape=True, + element_shape=None, + colocate_with_first_write_call=True, + name=None): + """Constructs an Eager mode TensorArray. + + Args: + dtype: (required) data type of the TensorArray. + size: (optional) int32 scalar `Tensor`: the size of the TensorArray. + Required if handle is not provided. + dynamic_size: (optional) Python bool: If true, writes to the TensorArray + can grow the TensorArray past its initial size. Default: False. + clear_after_read: Boolean (optional, default: True). If True, clear + TensorArray values after reading them. This disables read-many + semantics, but allows early release of memory. + tensor_array_name: unused. + handle: unsupported. + flow: unsupported. + infer_shape: used for error checking, same semantics as TensorArray. + element_shape: used for error checking, same semantics as TensorArray. + colocate_with_first_write_call: unsupported. + name: unsupported. + + Raises: + ValueError: handle or flow are supplied, or if size is not supplied. + """ + + del (flow, tensor_array_name, name) # not meaningful in Eager + + if handle is not None: + raise ValueError("TensorArray handles are not supported in Eager mode.") + if size is None: + raise ValueError("Size must be declared for TensorArrays in Eager mode.") + + # These attributes are not meaningful in Eager, but some library functions + # (e.g., those in control_flow_ops.py) access them to create new tensor + # arrays; as such, we define them for the sake of compatibility. + self._handle = None + # we assign a dummy value to _flow in case other code assumes it to be + # a Tensor + self._flow = constant_op.constant(0, dtype=dtypes.int32) + self._infer_shape = infer_shape + self._element_shape = element_shape + self._colocate_with_first_write_call = colocate_with_first_write_call + + self._dtype = dtype + self._dynamic_size = dynamic_size or False + self._clear_after_read = ( + True if clear_after_read is None else clear_after_read) + self._previously_read_indices = [] + + if isinstance(size, ops.EagerTensor): + size = size.numpy() + self._tensor_array = [None for _ in range(size)] + + @property + def flow(self): + """Flows are not meaningful in Eager; this exists for compatibility.""" + return self._flow + + @property + def dtype(self): + return self._dtype + + @property + def handle(self): + """Handles are not meaningful in Eager; this exists for compatibility.""" + return self._handle + + def _identity_without_array(self): + """Returns a new TensorArray with the same properties as this Eager one. + + NB: Does not set the underlying _tensor_array attribute. + """ + ta = TensorArray( + dtype=self._dtype, + size=len(self._tensor_array), + dynamic_size=self._dynamic_size, + clear_after_read=self._clear_after_read, + handle=self._handle, + flow=self._flow, + infer_shape=self._infer_shape, + element_shape=self._element_shape, + colocate_with_first_write_call=self._colocate_with_first_write_call) + ta._implementation._previously_read_indices = self._previously_read_indices # pylint: disable=protected-access + return ta + + def identity(self): + """See TensorArray.""" + ta = self._identity_without_array() + ta._implementation._tensor_array = [t for t in self._tensor_array] # pylint: disable=protected-access + return ta + + def grad(self, source, flow=None, name=None): + raise NotImplementedError( + "TensorArray.grad is not supported in Eager mode; Eager's gradient " + "implementation does not use/need this function to compute gradients " + "of operations that use TensorArrays.") + + def read(self, index, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + + if isinstance(index, ops.EagerTensor): + index = index.numpy() + + if index < 0: + raise errors_impl.OutOfRangeError( + None, None, + "Reading from negative indices (index %d) is not allowed." % index) + + if index >= len(self._tensor_array): + raise errors_impl.OutOfRangeError( + None, None, "Tried to read from index %d but array size is: %d" % + (index, len(self._tensor_array))) + + tensor = self._tensor_array[index] + if tensor is None: + if index in self._previously_read_indices: + raise errors_impl.InvalidArgumentError( + None, None, + "Could not read index %d twice because it was cleared after " + "a previous read (perhaps try setting clear_after_read = false?)" % + index) + else: + raise errors_impl.InvalidArgumentError( + None, None, + "Could not read from TensorArray index %d because it has not yet " + "been written to." % index) + + if self._clear_after_read: + self._tensor_array[index] = None + self._previously_read_indices.append(index) + return tensor + + def write(self, index, value, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + ta = self.identity() + _eager_write_no_copy(ta._implementation, index, value) # pylint: disable=protected-access + return ta + + def stack(self, name=None): + """See TensorArray.""" + try: + return array_ops.stack(self._tensor_array, name=name) + except ValueError: + if None in self._tensor_array: + idx = self._tensor_array.index(None) + raise errors_impl.InvalidArgumentError( + None, None, "Could not read from TensorArray index %d because " + "it has not yet been written to." % idx) + else: + raise + + def gather(self, indices, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + return array_ops.stack([self._tensor_array[i] for i in indices.numpy()]) + + def concat(self, name=None): + """See TensorArray.""" + try: + return array_ops.concat(self._tensor_array, 0, name=name) + except errors_impl.OpError: + # Reproduce a subset of the error-handling for graph-mode TensorArrays. + shapes = [t.shape for t in self._tensor_array] + ndims = [s.ndims for s in shapes] + if None in self._tensor_array: + # Concatenating empty TensorArrays is permitted if the element + # shape is defined; the output is a tensor with shape + # [0] + self._element_shape[1:] + if all(t is None for t in self._tensor_array): + if self._element_shape is not None: + return constant_op.constant([], shape=[0] + self._element_shape[1:]) + else: + raise errors_impl.UnimplementedError( + None, None, "TensorArray has size zero, but " + "element_shape_except0 %s is not fully defined. Currently only " + "static shapes are supported when concatenating zero-size " + "TensorArrays." % self._element_shape[1:]) + # Concatenating a TensorArray in which some but not all entries have + # been written to is not allowed. + idx = self._tensor_array.index(None) + raise errors_impl.InvalidArgumentError( + None, None, "Could not read from TensorArray index %d because " + "it has not yet been written to." % idx) + elif 0 in ndims: + idx = ndims.index(0) + raise errors_impl.InvalidArgumentError( + None, None, "Concat saw a scalar shape at index %d but requires " + "at least vectors." % idx) + else: + raise + + def unstack(self, value, name=None): + """See TensorArray.""" + tensors = array_ops.unstack(value, name=name) + if len(tensors) > len(self._tensor_array) and not self._dynamic_size: + raise ValueError( + "Cannot unstack %d tensors into a TensorArray of static size %d" % + (len(tensors), len(self._tensors))) + ta = self._identity_without_array() + ta._implementation._tensor_array = tensors # pylint: disable=protected-access + return ta + + def scatter(self, indices, value, name=None): + """See TensorArray.""" + del name # unused in Eager + ta = self.identity() + for index, val in zip(indices.numpy(), array_ops.unstack(value)): + _eager_write_no_copy(ta._implementation, index, val) # pylint: disable=protected-access + return ta + + def split(self, value, lengths, name=None): + """See TensorArray.""" + # error checking to match graph-mode errors + value = constant_op.constant(value) + lengths = constant_op.constant(lengths) + sum_lengths = math_ops.reduce_sum(lengths) + if lengths.shape.ndims != 1: + raise errors_impl.InvalidArgumentError( + None, None, "Expected lengths to be a vector, received shape: %s" % + lengths.shape.as_list()) + elif value.shape.ndims == 0: + raise errors_impl.InvalidArgumentError( + None, None, "Expected value to be at least a vector, " + "but received shape: %s" % value.shape.as_list()) + elif sum_lengths.numpy() != value.shape.as_list()[0]: + raise errors_impl.InvalidArgumentError( + None, None, "Expected sum of lengths to be equal to " + "values.shape[0], but sum of lengths is %d and " + "value's shape is: %s " % (sum_lengths.numpy(), + value.shape.as_list())) + elif not self._dynamic_size and lengths.shape[0] != len(self._tensor_array): + raise errors_impl.InvalidArgumentError( + None, None, "TensorArray's size is not equal to the size of " + "lengths (%d vs. %d), and the TensorArray is not marked as " + "dynamically resizeable" % (len(self._tensor_array), + lengths.shape[0])) + else: + ta = self._identity_without_array() + tensor_array = array_ops.split(value, lengths, name=name) + ta._implementation._tensor_array = tensor_array # pylint: disable=protected-access + return ta + + def size(self, name=None): + """See TensorArray.""" + del name # not meaningful in Eager mode + return constant_op.constant(len(self._tensor_array)) + + def close(self, name=None): + del name # not meaningful in Eager mode + del self._tensor_array[:] + return + + +# TensorArray is designed to hide an underlying implementation object +# and as such accesses many of that object's hidden fields. +# pylint: disable=protected-access +class TensorArray(object): + """Class wrapping dynamic-sized, per-time-step, write-once Tensor arrays. + + This class is meant to be used with dynamic iteration primitives such as + `while_loop` and `map_fn`. It supports gradient back-propagation via special + "flow" control flow dependencies. + """ + + def __init__(self, + dtype, + size=None, + dynamic_size=None, + clear_after_read=None, + tensor_array_name=None, + handle=None, + flow=None, + infer_shape=True, + element_shape=None, + colocate_with_first_write_call=True, + name=None): + """Construct a new TensorArray or wrap an existing TensorArray handle. + + A note about the parameter `name`: + + The name of the `TensorArray` (even if passed in) is uniquified: each time + a new `TensorArray` is created at runtime it is assigned its own name for + the duration of the run. This avoids name collisions if a `TensorArray` + is created within a `while_loop`. + + Args: + dtype: (required) data type of the TensorArray. + size: (optional) int32 scalar `Tensor`: the size of the TensorArray. + Required if handle is not provided. + dynamic_size: (optional) Python bool: If true, writes to the TensorArray + can grow the TensorArray past its initial size. Default: False. + clear_after_read: Boolean (optional, default: True). If True, clear + TensorArray values after reading them. This disables read-many + semantics, but allows early release of memory. + tensor_array_name: (optional) Python string: the name of the TensorArray. + This is used when creating the TensorArray handle. If this value is + set, handle should be None. + handle: (optional) A `Tensor` handle to an existing TensorArray. If this + is set, tensor_array_name should be None. Only supported in graph mode. + flow: (optional) A float `Tensor` scalar coming from an existing + `TensorArray.flow`. Only supported in graph mode. + infer_shape: (optional, default: True) If True, shape inference + is enabled. In this case, all elements must have the same shape. + element_shape: (optional, default: None) A `TensorShape` object specifying + the shape constraints of each of the elements of the TensorArray. + Need not be fully defined. + colocate_with_first_write_call: If `True`, the TensorArray will be + colocated on the same device as the Tensor used on its first write + (write operations include `write`, `unstack`, and `split`). If `False`, + the TensorArray will be placed on the device determined by the + device context available during its initialization. + name: A name for the operation (optional). + + Raises: + ValueError: if both handle and tensor_array_name are provided. + TypeError: if handle is provided but is not a Tensor. + """ + if context.in_graph_mode(): + implementation = _GraphTensorArray + else: + implementation = _EagerTensorArray + + self._implementation = implementation( + dtype, + size=size, + dynamic_size=dynamic_size, + clear_after_read=clear_after_read, + tensor_array_name=tensor_array_name, + handle=handle, + flow=flow, + infer_shape=infer_shape, + element_shape=element_shape, + colocate_with_first_write_call=colocate_with_first_write_call, + name=name) + + @property + def flow(self): + """The flow `Tensor` forcing ops leading to this TensorArray state.""" + return self._implementation._flow + + @property + def dtype(self): + """The data type of this TensorArray.""" + return self._implementation._dtype + + @property + def handle(self): + """The reference to the TensorArray.""" + return self._implementation._handle + + @property + def _infer_shape(self): + return self._implementation._infer_shape + + @_infer_shape.setter + def _infer_shape(self, infer_shape): + self._implementation._infer_shape = infer_shape + + @property + def _element_shape(self): + return self._implementation._element_shape + + @_element_shape.setter + def _element_shape(self, element_shape): + self._implementation._element_shape = element_shape + + @property + def _colocate_with_first_write_call(self): + return self._implementation._colocate_with_first_write_call + + @property + def _colocate_with(self): + return self._implementation._colocate_with + + @_colocate_with.setter + def _colocate_with(self, colocate_with): + self._implementation._colocate_with = colocate_with + + def identity(self): + """Returns a TensorArray with the same content and properties. + + Returns: + A new TensorArray object with flow that ensures the control dependencies + from the contexts will become control dependencies for writes, reads, etc. + Use this object all for subsequent operations. + """ + return self._implementation.identity() + + def grad(self, source, flow=None, name=None): + return self._implementation.grad(source, flow=flow, name=name) + + def read(self, index, name=None): + """Read the value at location `index` in the TensorArray. + + Args: + index: 0-D. int32 tensor with the index to read from. + name: A name for the operation (optional). + + Returns: + The tensor at index `index`. + """ + return self._implementation.read(index, name=name) + + @tf_should_use.should_use_result + def write(self, index, value, name=None): + """Write `value` into index `index` of the TensorArray. + + Args: + index: 0-D. int32 scalar with the index to write to. + value: N-D. Tensor of type `dtype`. The Tensor to write to this index. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the write occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if there are more writers than specified. + """ + return self._implementation.write(index, value, name=name) + + def stack(self, name=None): + """Return the values in the TensorArray as a stacked `Tensor`. + + All of the values must have been written and their shapes must all match. + If input shapes have rank-`R`, then output shape will have rank-`(R+1)`. + + Args: + name: A name for the operation (optional). + + Returns: + All the tensors in the TensorArray stacked into one tensor. + """ + return self._implementation.stack(name=name) + + def gather(self, indices, name=None): + """Return selected values in the TensorArray as a packed `Tensor`. + + All of selected values must have been written and their shapes + must all match. + + Args: + indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If + the `TensorArray` is not dynamic, `max_value=size()`. + name: A name for the operation (optional). + + Returns: + The tensors in the `TensorArray` selected by `indices`, packed into one + tensor. + """ + return self._implementation.gather(indices, name=name) + + def concat(self, name=None): + """Return the values in the TensorArray as a concatenated `Tensor`. + + All of the values must have been written, their ranks must match, and + and their shapes must all match for all dimensions except the first. + + Args: + name: A name for the operation (optional). + + Returns: + All the tensors in the TensorArray concatenated into one tensor. + """ + return self._implementation.concat(name=name) + + @tf_should_use.should_use_result + def unstack(self, value, name=None): + """Unstack the values of a `Tensor` in the TensorArray. + + If input value shapes have rank-`R`, then the output TensorArray will + contain elements whose shapes are rank-`(R-1)`. + + Args: + value: (N+1)-D. Tensor of type `dtype`. The Tensor to unstack. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the unstack occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if the shape inference fails. + """ + return self._implementation.unstack(value, name=name) + + @tf_should_use.should_use_result + def scatter(self, indices, value, name=None): + """Scatter the values of a `Tensor` in specific indices of a `TensorArray`. + + Args: + indices: A `1-D` `Tensor` taking values in `[0, max_value)`. If + the `TensorArray` is not dynamic, `max_value=size()`. + value: (N+1)-D. Tensor of type `dtype`. The Tensor to unpack. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the scatter occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if the shape inference fails. + """ + return self._implementation.scatter(indices, value, name=name) + + @tf_should_use.should_use_result + def split(self, value, lengths, name=None): + """Split the values of a `Tensor` into the TensorArray. + + Args: + value: (N+1)-D. Tensor of type `dtype`. The Tensor to split. + lengths: 1-D. int32 vector with the lengths to use when splitting + `value` along its first dimension. + name: A name for the operation (optional). + + Returns: + A new TensorArray object with flow that ensures the split occurs. + Use this object all for subsequent operations. + + Raises: + ValueError: if the shape inference fails. + """ + return self._implementation.split(value, lengths, name=name) + + def size(self, name=None): + """Return the size of the TensorArray.""" + return self._implementation.size(name=name) + + @tf_should_use.should_use_result + def close(self, name=None): + """Close the current TensorArray.""" + return self._implementation.close(name=name) + +# pylint: enable=protected-access diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py index 87805b5171d293f6134e351867fb44060b131ab3..92fa928eede1796df539f00751d7e419f5af8a9f 100644 --- a/tensorflow/python/ops/variable_scope.py +++ b/tensorflow/python/ops/variable_scope.py @@ -208,6 +208,7 @@ class _VariableStore(object): self._vars = {} # A dictionary of the stored TensorFlow variables. self._partitioned_vars = {} # A dict of the stored PartitionedVariables. self.variable_scopes_count = {} # Count re-used variable scopes. + self._store_eager_variables = False def open_variable_scope(self, scope_name): if scope_name in self.variable_scopes_count: @@ -259,8 +260,8 @@ class _VariableStore(object): applying it on a newly created variable will be added to the collection GraphKeys.REGULARIZATION_LOSSES and can be used for regularization. reuse: a Boolean, None, or tf.AUTO_REUSE. Controls reuse or creation - of variables. In Eager mode, this argument is always forced to be - tf.AUTO_REUSE. + of variables. When eager execution is enabled this argument is always + forced to be False. trainable: If `True` also add the variable to the graph collection `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`). collections: List of graph collections keys to add the `Variable` to. @@ -279,7 +280,8 @@ class _VariableStore(object): use_resource: If False, creates a regular Variable. If True, creates instead an experimental ResourceVariable which has well-defined semantics. Defaults to False (will later change to True). - In Eager mode, this argument is always forced to be true. + When eager execution is enabled this argument is always forced to be + true. custom_getter: Callable that takes as a first argument the true getter, and allows overwriting the internal get_variable method. The signature of `custom_getter` should match that of this method, @@ -308,13 +310,21 @@ class _VariableStore(object): ValueError: when creating a new variable and shape is not declared, when reusing a variable and specifying a conflicting shape, or when violating reuse during variable creation. + RuntimeError: when eager execution is enabled and not called from an + EagerVariableStore. """ if custom_getter is not None and not callable(custom_getter): raise ValueError( "Passed a custom_getter which is not callable: %s" % custom_getter) if context.in_eager_mode(): - reuse = AUTO_REUSE + if not self._store_eager_variables and reuse: + raise RuntimeError( + "When eager execution is enabled variable reuse is only supported" + " when an EagerVariableStore is active. See the documentation on" + " EagerVariableStore for example usage.") + if self._store_eager_variables: + reuse = AUTO_REUSE use_resource = True # If a *_ref type is passed in an error would be triggered further down the @@ -506,7 +516,7 @@ class _VariableStore(object): """ if context.in_eager_mode(): raise NotImplementedError("Partitioned variables are not yet supported " - "in Eager mode.") + "when eager execution is enabled.") initializing_from_value = initializer is not None and isinstance( initializer, ops.Tensor) @@ -580,7 +590,7 @@ class _VariableStore(object): if reuse is True: raise ValueError("PartitionedVariable %s does not exist, or was not " "created with tf.get_variable(). Did you mean to set " - "reuse=None in VarScope?" % name) + "reuse=False or reuse=tf.AUTO_REUSE in VarScope?" % name) slice_dim, slice_shape = _compute_slice_dim_and_shape( shape.as_list(), partitions) @@ -710,15 +720,6 @@ class _VariableStore(object): Raises: ValueError: See documentation of get_variable above. """ - # Fast-path for get_variable in eager mode when the variable already - # exists. Note this skips error validation code, so mismatched shapes and - # dtypes will be caught when the variable is used instead of when the call - # to get_variable happens. - if context.in_eager_mode(): - v = self._vars.get(name, None) - if v is not None: - return v - # Set to true if initializer is a constant. initializing_from_value = False if initializer is not None and not callable(initializer): @@ -803,7 +804,10 @@ class _VariableStore(object): dtype=variable_dtype, validate_shape=validate_shape, constraint=constraint) - self._vars[name] = v + if context.in_graph_mode() or self._store_eager_variables: + # In eager mode we do not want to keep default references to Variable + # objects as this will prevent their memory from being released. + self._vars[name] = v logging.vlog(1, "Created variable %s with shape %s and init %s", v.name, format(shape), initializer) @@ -875,8 +879,8 @@ class VariableScope(object): initializer: default initializer passed to get_variable. regularizer: default regularizer passed to get_variable. reuse: Boolean, None, or tf.AUTO_REUSE, setting the reuse in - get_variable. In Eager mode, this argument is always forced to be - tf.AUTO_REUSE. + get_variable. When eager execution is enabled this argument is always + forced to be False. caching_device: string, callable, or None: the caching device passed to get_variable. partitioner: callable or `None`: the partitioner passed to `get_variable`. @@ -885,8 +889,8 @@ class VariableScope(object): dtype: default type passed to get_variable (defaults to DT_FLOAT). use_resource: if False, create a normal Variable; if True create an experimental ResourceVariable with well-defined semantics. Defaults - to False (will later change to True). In Eager mode, this argument is - always forced to be True. + to False (will later change to True). When eager execution is enabled + this argument is always forced to be True. constraint: An optional projection function to be applied to the variable after being updated by an `Optimizer` (e.g. used to implement norm constraints or value constraints for layer weights). The function must @@ -923,10 +927,10 @@ class VariableScope(object): if context.in_eager_mode(): if self._caching_device is not None: raise NotImplementedError("Caching devices is not yet supported " - "in Eager mode.") + "when eager execution is enabled.") if self._partitioner is not None: raise NotImplementedError("Partitioned variables are not yet supported " - "in Eager mode.") + "when eager execution is enabled.") self._reuse = AUTO_REUSE self._use_resource = True @@ -989,7 +993,8 @@ class VariableScope(object): def set_use_resource(self, use_resource): """Sets whether to use ResourceVariables for this scope.""" if context.in_eager_mode() and not use_resource: - raise ValueError("In eager mode, use_resource cannot be set to false.") + raise ValueError("When eager execution is enabled, " + "use_resource cannot be set to false.") self._use_resource = use_resource def set_regularizer(self, regularizer): @@ -999,15 +1004,15 @@ class VariableScope(object): def set_caching_device(self, caching_device): """Set caching_device for this scope.""" if context.in_eager_mode(): - raise NotImplementedError("Partitioned variables are not yet supported " - "in Eager mode.") + raise NotImplementedError("Caching devices are not yet supported " + "when eager execution is enabled.") self._caching_device = caching_device def set_partitioner(self, partitioner): """Set partitioner for this scope.""" if partitioner and context.in_eager_mode(): raise NotImplementedError("Partitioned variables are not yet supported " - "in Eager mode.") + "when eager execution is enabled.") self._partitioner = partitioner def set_custom_getter(self, custom_getter): @@ -1062,7 +1067,7 @@ class VariableScope(object): if use_resource is None: use_resource = self._use_resource else: - reuse = AUTO_REUSE + reuse = False use_resource = True full_name = self.name + "/" + name if self.name else name @@ -1108,7 +1113,7 @@ class VariableScope(object): """Gets an existing variable with this name or create a new one.""" if context.in_eager_mode(): raise NotImplementedError("Partitioned variables are not yet supported " - "in Eager mode.") + "when eager execution is enabled.") if initializer is None: initializer = self._initializer if regularizer is None: @@ -1181,6 +1186,48 @@ def _get_default_variable_store(): return store +@tf_contextlib.contextmanager +def with_variable_store(store): + store_collection = ops.get_collection_ref(_VARSTORE_KEY) + old = list(store_collection) + store_collection[:] = [store] + try: + yield + finally: + store_collection[:] = old + + +class EagerVariableStore(object): + """Wrapper allowing functional layers to be used with eager execution. + + When eager execution is enabled Variables get deleted when they go out of + scope, and are not stored in global collections by default. A lot of code + (mostly the functional layers in tf.layers) assumes that variables are kept in + a global list. + + EagerVariableStore can be used in conjunction with this code to make it + eager-friendly. For example, to create a dense layer, use: + + ``` + container = tfe.EagerVariableStore() + for input in dataset_iterator: + with container.as_default(): + x = tf.layers.dense(input, name="l1") + print(container.variables) # Should print the variables used in the layer. + ``` + """ + + def __init__(self): + self._store = _VariableStore() + self._store._store_eager_variables = True # pylint: disable=protected-access + + def as_default(self): + return with_variable_store(self._store) + + def variables(self): + return self._store._vars.values() # pylint: disable=protected-access + + def get_variable(name, shape=None, dtype=None, @@ -1259,8 +1306,8 @@ Args: must be known. use_resource: If False, creates a regular Variable. If true, creates an experimental ResourceVariable instead with well-defined semantics. - Defaults to False (will later change to True). In Eager mode, this argument - is always forced to be True. + Defaults to False (will later change to True). When eager execution is + enabled this argument is always forced to be True. custom_getter: Callable that takes as a first argument the true getter, and allows overwriting the internal get_variable method. The signature of `custom_getter` should match that of this method, @@ -1721,14 +1768,14 @@ class variable_scope(object): # pylint: disable=invalid-name reuse: `True`, None, or tf.AUTO_REUSE; if `True`, we go into reuse mode for this scope as well as all sub-scopes; if tf.AUTO_REUSE, we create variables if they do not exist, and return them otherwise; if None, we - inherit the parent scope's reuse flag. In Eager mode, this argument is - always forced to be tf.AUTO_REUSE. + inherit the parent scope's reuse flag. When eager execution is enabled, + this argument is always forced to be tf.AUTO_REUSE. dtype: type of variables created in this scope (defaults to the type in the passed scope, or inherited from parent scope). use_resource: If False, all variables will be regular Variables. If True, experimental ResourceVariables with well-defined semantics will be used - instead. Defaults to False (will later change to True). In Eager mode, - this argument is always forced to be True. + instead. Defaults to False (will later change to True). When eager + execution is enabled this argument is always forced to be True. constraint: An optional projection function to be applied to the variable after being updated by an `Optimizer` (e.g. used to implement norm constraints or value constraints for layer weights). The function must @@ -1927,11 +1974,17 @@ def variable(initial_value=None, caching_device=None, name=None, dtype=None): - if get_variable_scope().use_resource: + use_resource = get_variable_scope().use_resource + if use_resource or (use_resource is None and context.in_eager_mode()): return resource_variable_ops.ResourceVariable( initial_value=initial_value, trainable=trainable, collections=collections, validate_shape=validate_shape, caching_device=caching_device, name=name, dtype=dtype) + elif not use_resource and context.in_eager_mode(): + raise RuntimeError( + "VariableScope should use resource variable when eager execution is" + " enabled, but use_resource is False." + ) else: return variables.Variable( initial_value=initial_value, trainable=trainable, diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py index 90b4f25d81af3f98a90e0f28f4e62876837daf92..f906b7b3c47b218cb789f96d8f258e0644e0dbe3 100644 --- a/tensorflow/python/ops/variables.py +++ b/tensorflow/python/ops/variables.py @@ -118,6 +118,14 @@ class Variable(object): `trainable_variables()` returns the contents of this collection. The various `Optimizer` classes use this collection as the default list of variables to optimize. + + @compatibility(eager) + `tf.Variable` is not compatible with eager execution. Use + `tfe.Variable` instead which is compatible with both eager execution + and graph construction. See [the TensorFlow Eager Execution + guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) + for details on how variables work in eager execution. + @end_compatibility """ def __init__(self, @@ -188,11 +196,19 @@ class Variable(object): ValueError: If both `variable_def` and initial_value are specified. ValueError: If the initial value is not specified, or does not have a shape and `validate_shape` is `True`. - RuntimeError: If created in EAGER mode. + RuntimeError: If eager execution is enabled. + + @compatibility(eager) + `tf.Variable` is not compatible with eager execution. Use + `tfe.Variable` instead which is compatable with both eager execution + and graph construction. See [the TensorFlow Eager Execution + guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) + for details on how variables work in eager execution. + @end_compatibility """ if not context.in_graph_mode(): - raise RuntimeError("Variable not supported in Eager mode. " - "Please use ResourceVariable instead") + raise RuntimeError("tf.Variable not supported in Eager mode. " + "Please use tfe.Variable instead") if variable_def: # If variable_def is provided, recreates the variable from its fields. if initial_value: @@ -394,7 +410,8 @@ class Variable(object): import_scope=import_scope)) if variable_def.HasField("save_slice_info_def"): self._save_slice_info = Variable.SaveSliceInfo( - save_slice_info_def=variable_def.save_slice_info_def) + save_slice_info_def=variable_def.save_slice_info_def, + import_scope=import_scope) else: self._save_slice_info = None self._caching_device = None @@ -1044,7 +1061,16 @@ class Variable(object): class PartitionedVariable(object): - """A container for partitioned `Variable` objects.""" + """A container for partitioned `Variable` objects. + + @compatiblity(eager) `tf.PartitionedVariable` is not compatible with + eager execution. Use `tfe.Variable` instead which is compatable + with both eager execution and graph construction. See [the + TensorFlow Eager Execution + guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers) + for details on how variables work in eager execution. + @end_compatiblity + """ class PartitionedVariableIterator(object): """An iterator that allows accessing the underlying `Variable` objects. @@ -1093,10 +1119,11 @@ class PartitionedVariable(object): `partitions` is not a list. ValueError: If `variable_list` is empty, or the `Variable` shape information does not match `shape`, or `partitions` has invalid values. - RuntimeError: If created in EAGER mode. + RuntimeError: If eager execution is enabled """ if not context.in_graph_mode(): - raise RuntimeError("PartitionedVariable not supported in Eager mode.") + raise RuntimeError("tf.PartitionedVariable not supported in " + "eager mode. Please use tfe.Variable instead") if not isinstance(variable_list, (list, tuple)): raise TypeError( "variable_list is not a list or tuple: %s" % variable_list) diff --git a/tensorflow/python/platform/sysconfig.py b/tensorflow/python/platform/sysconfig.py index 1e3be40933b6165160e5b3e90ea87a63ef201c32..167dec6551f9321d01732ab4264fdb28a7bb6916 100644 --- a/tensorflow/python/platform/sysconfig.py +++ b/tensorflow/python/platform/sysconfig.py @@ -17,6 +17,8 @@ @@get_include @@get_lib +@@get_compile_flags +@@get_link_flags """ from __future__ import absolute_import from __future__ import division @@ -24,6 +26,7 @@ from __future__ import print_function import os.path as _os_path +from tensorflow.python.framework.versions import CXX11_ABI_FLAG as _CXX11_ABI_FLAG from tensorflow.python.util.all_util import remove_undocumented @@ -51,5 +54,31 @@ def get_lib(): import tensorflow as tf return _os_path.join(_os_path.dirname(tf.__file__)) + +def get_compile_flags(): + """Get the compilation flags for custom operators. + + Returns: + The compilation flags. + """ + flags = [] + flags.append('-I%s' % get_include()) + flags.append('-I%s/external/nsync/public' % get_include()) + if _CXX11_ABI_FLAG != -1: + flags.append('-D_GLIBCXX_USE_CXX11_ABI=%d' % _CXX11_ABI_FLAG) + return flags + + +def get_link_flags(): + """Get the link flags for custom operators. + + Returns: + The link flags. + """ + flags = [] + flags.append('-L%s' % get_lib()) + flags.append('-ltensorflow_framework') + return flags + _allowed_symbols = [] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/profiler/internal/flops_registry.py b/tensorflow/python/profiler/internal/flops_registry.py index e143501049d19e2e0d679f7bc1810443decb0a11..147711b1d9b864c195f17b50eb3e7bc37ee1ecd0 100644 --- a/tensorflow/python/profiler/internal/flops_registry.py +++ b/tensorflow/python/profiler/internal/flops_registry.py @@ -373,6 +373,7 @@ def _max_pool_grad_flops(graph, node): kernel_area = _list_product(kernel_shape) orig_out_shape = graph_util.tensor_shape_from_node_def_name(graph, node.input[1]) + orig_out_shape.assert_is_fully_defined() max_pool_ops = kernel_area * orig_out_shape.num_elements() return ops.OpStats("flops", max_pool_ops + orig_out_shape.num_elements()) diff --git a/tensorflow/python/profiler/model_analyzer.py b/tensorflow/python/profiler/model_analyzer.py index 2071325c7bb87eca80e07a74e27683911640af2c..040a4891637109590acbc8a71c11e0d863a34c11 100644 --- a/tensorflow/python/profiler/model_analyzer.py +++ b/tensorflow/python/profiler/model_analyzer.py @@ -157,6 +157,7 @@ class Profiler(object): op_log: optional. tensorflow::tfprof::OpLogProto proto. Used to define extra op types. """ + self._coverage = 0.0 self._graph = graph # pylint: disable=protected-access op_log = tfprof_logger._merge_default_with_oplog( @@ -183,7 +184,7 @@ class Profiler(object): self._graph, run_meta=run_meta) # pylint: enable=protected-access # TODO(xpan): P1: Better to find the current graph. - print_mdl.AddStep( + self._coverage = print_mdl.AddStep( step, self._graph.as_graph_def(add_shapes=True).SerializeToString(), run_meta.SerializeToString(), op_log.SerializeToString()) @@ -274,6 +275,10 @@ class Profiler(object): print_mdl.Profile('advise'.encode('utf-8'), opts.SerializeToString())) return advise_pb + def _write_profile(self, filename): + """Writes the profile to a file.""" + print_mdl.WriteProfile(filename) + def profile(graph, run_meta=None, diff --git a/tensorflow/python/profiler/model_analyzer_test.py b/tensorflow/python/profiler/model_analyzer_test.py index 2578fc3e87fc6ac3ea0c5194b461e119556d8036..17c87bea92dedf3f04e2f4e151e45610d27e34ef 100644 --- a/tensorflow/python/profiler/model_analyzer_test.py +++ b/tensorflow/python/profiler/model_analyzer_test.py @@ -159,7 +159,7 @@ class PrintModelAnalysisTest(test.TestCase): with gfile.Open(outfile, 'r') as f: # pylint: disable=line-too-long self.assertEqual( - 'node name | # parameters | # float_ops | assigned devices | op types | op count (run|defined) | input shapes\n_TFProfRoot (--/451 params, --/11.34k flops, _kTFScopeParent, --/8|--/36, )\n Conv2D (0/0 params, 5.83k/5.83k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x6x6x3|1:3x3x3x6)\n Conv2D_1 (0/0 params, 4.61k/4.61k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x3x3x6|1:2x2x6x12)\n DW (3x3x3x6, 162/162 params, 0/324 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:3x3x3x6|1:3x3x3x6)\n DW/Initializer (0/0 params, 0/324 flops, _kTFScopeParent, 0/0|1/7, )\n DW/Initializer/random_normal (0/0 params, 162/324 flops, Add, 0/0|1/6, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/mul (0/0 params, 162/162 flops, Mul, 0/0|1/1, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:3x3x3x6)\n DW2 (2x2x6x12, 288/288 params, 0/576 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW2/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:2x2x6x12|1:2x2x6x12)\n DW2/Initializer (0/0 params, 0/576 flops, _kTFScopeParent, 0/0|1/7, )\n DW2/Initializer/random_normal (0/0 params, 288/576 flops, Add, 0/0|1/6, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW2/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/mul (0/0 params, 288/288 flops, Mul, 0/0|1/1, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:2x2x6x12)\n ScalarW (1, 1/1 params, 0/2 flops, VariableV2|_trainable_variables, 0/0|1/10, )\n ScalarW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer (0/0 params, 0/2 flops, _kTFScopeParent, 0/0|1/7, )\n ScalarW/Initializer/random_normal (0/0 params, 1/2 flops, Add, 0/0|1/6, 0:1|1:1)\n ScalarW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:0)\n ScalarW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/mul (0/0 params, 1/1 flops, Mul, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/read (0/0 params, 0/0 flops, Identity, 0/0|1/1, 0:1)\n _retval_Conv2D_1_0_0 (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|RunTimeOp, 1/1|1/1, )\n init (0/0 params, 0/0 flops, NoOp, 0/0|1/1, 0:1|1:3x3x3x6|2:2x2x6x12)\n zeros (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Const, 1/1|1/1, )\n', + 'node name | # parameters | # float_ops | assigned devices | op types | op count (run|defined) | input shapes\n_TFProfRoot (--/451 params, --/11.34k flops, _kTFScopeParent, --/8|--/36, )\n Conv2D (0/0 params, 5.83k/5.83k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x6x6x3|1:3x3x3x6)\n Conv2D_1 (0/0 params, 4.61k/4.61k flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Conv2D, 1/1|1/1, 0:2x3x3x6|1:2x2x6x12)\n DW (3x3x3x6, 162/162 params, 0/324 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:3x3x3x6|1:3x3x3x6)\n DW/Initializer (0/0 params, 0/324 flops, _kTFScopeParent, 0/0|1/7, )\n DW/Initializer/random_normal (0/0 params, 162/324 flops, Add, 0/0|1/6, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/mul (0/0 params, 162/162 flops, Mul, 0/0|1/1, 0:3x3x3x6|1:1)\n DW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:3x3x3x6)\n DW2 (2x2x6x12, 288/288 params, 0/576 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|VariableV2|_trainable_variables, 1/2|1/10, )\n DW2/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:2x2x6x12|1:2x2x6x12)\n DW2/Initializer (0/0 params, 0/576 flops, _kTFScopeParent, 0/0|1/7, )\n DW2/Initializer/random_normal (0/0 params, 288/576 flops, Add, 0/0|1/6, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:4)\n DW2/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/mul (0/0 params, 288/288 flops, Mul, 0/0|1/1, 0:2x2x6x12|1:1)\n DW2/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n DW2/read (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Identity, 1/1|1/1, 0:2x2x6x12)\n ScalarW (1, 1/1 params, 0/2 flops, VariableV2|_trainable_variables, 0/0|1/10, )\n ScalarW/Assign (0/0 params, 0/0 flops, Assign, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer (0/0 params, 0/2 flops, _kTFScopeParent, 0/0|1/7, )\n ScalarW/Initializer/random_normal (0/0 params, 1/2 flops, Add, 0/0|1/6, 0:1|1:1)\n ScalarW/Initializer/random_normal/RandomStandardNormal (0/0 params, 0/0 flops, RandomStandardNormal, 0/0|1/1, 0:0)\n ScalarW/Initializer/random_normal/mean (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/mul (0/0 params, 1/1 flops, Mul, 0/0|1/1, 0:1|1:1)\n ScalarW/Initializer/random_normal/shape (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/Initializer/random_normal/stddev (0/0 params, 0/0 flops, Const, 0/0|1/1, )\n ScalarW/read (0/0 params, 0/0 flops, Identity, 0/0|1/1, 0:1)\n _retval_Conv2D_1_0_0 (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|_retval_Conv2D_1_0_0, 1/1|1/1, )\n init (0/0 params, 0/0 flops, NoOp, 0/0|1/1, 0:1|1:3x3x3x6|2:2x2x6x12)\n zeros (0/0 params, 0/0 flops, /job:localhost/replica:0/task:0/device:cpu:0, /job:localhost/replica:0/task:0/device:cpu:0|Const, 1/1|1/1, )\n', f.read()) # pylint: enable=line-too-long diff --git a/tensorflow/python/profiler/profile_context.py b/tensorflow/python/profiler/profile_context.py index 0c31cf8f134d1ba411b92da5daf6a8090f34659c..c7c7ad63012a153d41aa9d616dbd39acb46096f6 100644 --- a/tensorflow/python/profiler/profile_context.py +++ b/tensorflow/python/profiler/profile_context.py @@ -20,6 +20,8 @@ from __future__ import print_function import contextlib import os +import random +import sys import threading from tensorflow.core.protobuf import config_pb2 @@ -31,6 +33,7 @@ from tensorflow.python.platform import gfile from tensorflow.python.profiler import model_analyzer from tensorflow.python.util import compat +WARMUP_STEPS = 10 MAX_TRACED_STEPS = 100 @@ -51,7 +54,9 @@ def _profiled_run(self, # Fast path if no need for profiling. if not self.profile_context._is_fast_path(): # Maybe trace this step. - if self.profile_context._should_trace(): + if self.profile_context._should_trace(self.graph, fetches): + if self.profile_context._debug: + sys.stderr.write('debug: tracing step: %d\n' % step) # Enable tracing, perform auto profiling or auto dump. if not run_metadata: run_metadata = config_pb2.RunMetadata() @@ -66,6 +71,8 @@ def _profiled_run(self, ret = self._profiler_run_internal( fetches, feed_dict, options, run_metadata) + if self.profile_context._debug: + self.profile_context._dump_file(run_metadata, 'run_meta_%d' % step) self.profile_context.profiler._graph = self.graph self.profile_context.profiler.add_step(step, run_metadata) @@ -80,6 +87,8 @@ def _profiled_run(self, to_profiles = self.profile_context._profile_candidates() for to_prof in to_profiles: cmd, opts, _ = to_prof + if self.profile_context._debug: + sys.stderr.write('debug: profiling %s step: %d\n' % (cmd, step)) if cmd == 'graph': self.profile_context.profiler.profile_graph(opts) elif cmd == 'scope': @@ -131,29 +140,43 @@ class ProfileContext(object): pre-defined steps. dump_steps: A list of steps to dump the profile to `profile_dir`. If None, use pre-defined steps. + enabled: If false, everything is disabled with minimal overhead. It allows + user to only enable profiling when needed. + debug: If true, also dumps the raw trace RunMetadata text file to + profile_dir. And print debugging message. Useful for bug report. """ def __init__(self, profile_dir, trace_steps=None, - dump_steps=None): + dump_steps=None, + enabled=True, + debug=False): + self._enabled = enabled + if not self._enabled: + return + + self._debug = debug if not profile_dir: raise ValueError('Must have a directory for profile.\n') self._profiler_dir = profile_dir if trace_steps is None: - self._trace_steps = set(list(range(10, 100, 3)) + - list(range(100, 10000, 1000))) + self._trace_steps = set() + self._auto_tracing = True else: if len(trace_steps) > MAX_TRACED_STEPS: raise ValueError('Only support tracing up to 100 steps.\n') self._trace_steps = set(trace_steps[:]) + self._auto_tracing = False if dump_steps is None: - self._dump_steps = set([100] + list(range(100, 10000, 2000))) + self._dump_steps = set([MAX_TRACED_STEPS]) else: self._dump_steps = set(dump_steps[:]) + self._rng = random.Random(111) + self._fetched = set() self._slow_path_steps = self._dump_steps | self._trace_steps self._trace_next_step = False self._dump_next_step = False @@ -173,6 +196,8 @@ class ProfileContext(object): will be run automatically at these integer steps. Each step is a session.run. """ + if not self._enabled: + return self._auto_profiles.append((cmd, options, profile_steps[:])) self._slow_path_steps |= set(profile_steps) self._trace_steps |= set(profile_steps) @@ -180,41 +205,82 @@ class ProfileContext(object): @property def profiler(self): """Returns the current profiler object.""" + if not self._enabled: + return None if not self._profiler: self._profiler = model_analyzer.Profiler(ops.get_default_graph()) return self._profiler def trace_next_step(self): - """Enables tracing and add traces to profiler at next step.""" + """Enables tracing and adds traces to profiler at next step.""" + if not self._enabled: + return self._trace_next_step = True + self._slow_path_steps.add(self._step) def dump_next_step(self): """Enable tracing and dump profiles at next step.""" + if not self._enabled: + return self._dump_next_step = True + self._slow_path_steps.add(self._step) def _is_fast_path(self): - if (self._step in self._slow_path_steps or - self._trace_next_step or - self._dump_next_step): + if self._step in self._slow_path_steps: + return False + # When user doesn't set the tracing steps explicitly, auto decide it. + if (self._auto_tracing and self._step > WARMUP_STEPS and + self._traced_steps <= MAX_TRACED_STEPS): return False return True - def _should_trace(self): + def _should_trace(self, graph, fetches): + """Whether should do tracing at current step.""" if self._traced_steps > MAX_TRACED_STEPS: return False - trace = self._step in self._trace_steps or self._trace_next_step - if trace: + # Check user-set tracing steps. + if self._step in self._trace_steps or self._trace_next_step: self._traced_steps += 1 - return trace + return True + + # If no user-set tracing steps set and passes warm up steps, auto trace. + if self._auto_tracing and self._step > WARMUP_STEPS: + # If the fetches have not been seen before, trace it. + with graph.as_default(): + fetch_names = [f.name for f in + session._FetchMapper.for_fetch(fetches).unique_fetches()] # pylint: disable=protected-access + fetch_name = '-'.join(sorted(fetch_names)) + if self._debug: + sys.stderr.write('debug: trace fetches: %s\n' % fetch_name) + if fetch_name not in self._fetched: + self._fetched.add(fetch_name) + self._traced_steps += 1 + return True + # If the trace coverage is low, does some random tracing. + if (self.profiler._coverage < 0.5 and self._step < MAX_TRACED_STEPS and # pylint: disable=protected-access + self._rng.randint(0, 10) < 2): + self._traced_steps += 1 + return True + return False def _maybe_dump(self): + """Maybe dump the profile file.""" if not (self._step in self._dump_steps or self._dump_next_step): return + if self._debug: + sys.stderr.write('debug: dumping file at step: %d\n' % self._step) if not gfile.Exists(self._profiler_dir): gfile.MakeDirs(self._profiler_dir) - print_mdl.WriteProfile( - os.path.join(compat.as_bytes(self._profiler_dir), - compat.as_bytes('profile_%d' % self._step))) + + filename = os.path.join(compat.as_bytes(self._profiler_dir), + compat.as_bytes('profile_%d' % self._step)) + self.profiler._write_profile(filename) # pylint: disable=protected-access + + def _dump_file(self, pb, basename): + if not gfile.Exists(self._profiler_dir): + gfile.MakeDirs(self._profiler_dir) + with gfile.Open(os.path.join(self._profiler_dir, basename), 'w') as f: + f.write('%s' % pb) @contextlib.contextmanager def _new_step(self): @@ -233,28 +299,33 @@ class ProfileContext(object): return to_profile def __enter__(self): - self.old_run = getattr(session.BaseSession, 'run', None) - self.old_init = getattr(session.BaseSession, '__init__', None) - if not self.old_run: - raise errors.InternalError(None, None, 'BaseSession misses run method.') - elif not self.old_init: - raise errors.InternalError(None, None, - 'BaseSession misses __init__ method.') - elif getattr(session.BaseSession, '_profiler_run_internal', None): - raise errors.InternalError(None, None, - 'Already in context or context not cleaned.') - elif getattr(session.BaseSession, '_profiler_init_internal', None): - raise errors.InternalError(None, None, - 'Already in context or context not cleaned.') + if self._enabled: + self.old_run = getattr(session.BaseSession, 'run', None) + self.old_init = getattr(session.BaseSession, '__init__', None) + if not self.old_run: + raise errors.InternalError(None, None, 'BaseSession misses run method.') + elif not self.old_init: + raise errors.InternalError(None, None, + 'BaseSession misses __init__ method.') + elif getattr(session.BaseSession, '_profiler_run_internal', None): + raise errors.InternalError(None, None, + 'Already in context or context not cleaned.') + elif getattr(session.BaseSession, '_profiler_init_internal', None): + raise errors.InternalError(None, None, + 'Already in context or context not cleaned.') + else: + setattr(session.BaseSession, 'run', _profiled_run) + setattr(session.BaseSession, '__init__', _profiled_init) + setattr(session.BaseSession, '_profiler_run_internal', self.old_run) + setattr(session.BaseSession, '_profiler_init_internal', self.old_init) + setattr(session.BaseSession, 'profile_context', self) + return self else: - setattr(session.BaseSession, 'run', _profiled_run) - setattr(session.BaseSession, '__init__', _profiled_init) - setattr(session.BaseSession, '_profiler_run_internal', self.old_run) - setattr(session.BaseSession, '_profiler_init_internal', self.old_init) - setattr(session.BaseSession, 'profile_context', self) return self def __exit__(self, exec_type, exec_value, exec_tb): + if not self._enabled: + return print_mdl.DeleteProfiler() setattr(session.BaseSession, 'run', self.old_run) setattr(session.BaseSession, '__init__', self.old_init) diff --git a/tensorflow/python/profiler/profile_context_test.py b/tensorflow/python/profiler/profile_context_test.py index bbb49974ed343d0a8ab22a5f4c8d49bf0c82200c..a623beee23ebf98cf96bd0f334f813db5ae04040 100644 --- a/tensorflow/python/profiler/profile_context_test.py +++ b/tensorflow/python/profiler/profile_context_test.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function import os + from tensorflow.python.client import session from tensorflow.python.framework import ops from tensorflow.python.ops import variables @@ -66,6 +67,49 @@ class ProfilerContextTest(test.TestCase): with gfile.Open(outfile, "r") as f: self.assertEqual(profile_str, f.read()) + def testAutoTracingInDeubMode(self): + ops.reset_default_graph() + x = lib.BuildFullModel() + + with profile_context.ProfileContext(test.get_temp_dir(), debug=True): + with session.Session() as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(10): + sess.run(x) + for f in gfile.ListDirectory(test.get_temp_dir()): + # Warm up, no tracing. + self.assertFalse("run_meta" in f) + sess.run(x) + self.assertTrue( + gfile.Exists(os.path.join(test.get_temp_dir(), "run_meta_11"))) + gfile.Remove(os.path.join(test.get_temp_dir(), "run_meta_11")) + # fetched already. + sess.run(x) + for f in gfile.ListDirectory(test.get_temp_dir()): + self.assertFalse("run_meta" in f) + + def testDisabled(self): + ops.reset_default_graph() + x = lib.BuildFullModel() + with profile_context.ProfileContext(test.get_temp_dir(), + enabled=False) as pctx: + with session.Session() as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(10): + sess.run(x) + self.assertTrue(pctx.profiler is None) + self.assertTrue( + getattr(session.BaseSession, "profile_context", None) is None) + + with profile_context.ProfileContext(test.get_temp_dir()) as pctx: + with session.Session() as sess: + sess.run(variables.global_variables_initializer()) + for _ in range(10): + sess.run(x) + self.assertFalse(pctx.profiler is None) + self.assertFalse( + getattr(session.BaseSession, "profile_context", None) is None) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/pywrap_tensorflow.py b/tensorflow/python/pywrap_tensorflow.py index 000ed8df8b553f3841f0910d3b2d04db419b2601..91373fa544b62e1b4760a92bf6630edf0c7f1ee4 100644 --- a/tensorflow/python/pywrap_tensorflow.py +++ b/tensorflow/python/pywrap_tensorflow.py @@ -59,6 +59,7 @@ try: from tensorflow.python.pywrap_tensorflow_internal import __version__ from tensorflow.python.pywrap_tensorflow_internal import __git_version__ from tensorflow.python.pywrap_tensorflow_internal import __compiler_version__ + from tensorflow.python.pywrap_tensorflow_internal import __cxx11_abi_flag__ if _use_dlopen_global_flags: pywrap_dlopen_global_flags.reset_dlopen_flags() diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i index 5c624a9c126e6b2cfaccc4da8e7acbd4e325bb64..fa36b77311e277e1b17b4ee70da3bcf98b65bd1e 100644 --- a/tensorflow/python/pywrap_tfe.i +++ b/tensorflow/python/pywrap_tfe.i @@ -30,12 +30,26 @@ limitations under the License. %rename("%s") TFE_Py_TapeDeleteTrace; %rename("%s") TFE_Py_TapeRecordOperation; %rename("%s") TFE_Py_TapeExport; - +%rename("%s") TFE_NewContextOptions; +%rename("%s") TFE_ContextOptionsSetConfig; +%rename("%s") TFE_ContextOptionsSetDevicePlacementPolicy; +%rename("%s") TFE_DeleteContextOptions; %{ #include "tensorflow/python/eager/pywrap_tfe.h" %} +%typemap(in) (const void* proto) { + char* c_string; + Py_ssize_t py_size; + // PyBytes_AsStringAndSize() does not copy but simply interprets the input + if (PyBytes_AsStringAndSize($input, &c_string, &py_size) == -1) { + // Python has raised an error (likely TypeError or UnicodeEncodeError). + SWIG_fail; + } + $1 = static_cast(c_string); +} + %typemap(out) TF_DataType { $result = PyInt_FromLong($1); } @@ -88,6 +102,11 @@ limitations under the License. } } +%rename("%s") TFE_ContextDevicePlacementPolicy; +%rename("%s") TFE_DEVICE_PLACEMENT_EXPLICIT; +%rename("%s") TFE_DEVICE_PLACEMENT_WARN; +%rename("%s") TFE_DEVICE_PLACEMENT_SILENT; + %include "tensorflow/c/eager/c_api.h" %typemap(in) TFE_InputTensorHandles* inputs (TFE_InputTensorHandles temp) { @@ -165,3 +184,4 @@ limitations under the License. %typemap(in, numinputs=0) TF_Status *out_status; %typemap(freearg) (TF_Status* out_status); %typemap(argout) (TFE_OutputTensorHandles* outputs, TF_Status* out_status); +%typemap(in) (const void* proto); diff --git a/tensorflow/python/saved_model/signature_def_utils_impl.py b/tensorflow/python/saved_model/signature_def_utils_impl.py index 564befeb0b56146fee169cbcd031f0d5ce3e1a82..240ea61aa5f8553852044f84b61d010bfbca69d1 100644 --- a/tensorflow/python/saved_model/signature_def_utils_impl.py +++ b/tensorflow/python/saved_model/signature_def_utils_impl.py @@ -56,9 +56,13 @@ def build_signature_def(inputs=None, outputs=None, method_name=None): def regression_signature_def(examples, predictions): """Creates regression signature from given examples and predictions. + This function produces signatures intended for use with the TensorFlow Serving + Regress API (tensorflow_serving/apis/prediction_service.proto), and so + constrains the input and output types to those allowed by TensorFlow Serving. + Args: - examples: `Tensor`. - predictions: `Tensor`. + examples: A string `Tensor`, expected to accept serialized tf.Examples. + predictions: A float `Tensor`. Returns: A regression-flavored signature_def. @@ -93,10 +97,15 @@ def regression_signature_def(examples, predictions): def classification_signature_def(examples, classes, scores): """Creates classification signature from given examples and predictions. + This function produces signatures intended for use with the TensorFlow Serving + Classify API (tensorflow_serving/apis/prediction_service.proto), and so + constrains the input and output types to those allowed by TensorFlow Serving. + Args: - examples: `Tensor`. - classes: `Tensor`. - scores: `Tensor`. + examples: A string `Tensor`, expected to accept serialized tf.Examples. + classes: A string `Tensor`. Note that the ClassificationResponse message + requires that class labels are strings, not integers or anything else. + scores: a float `Tensor`. Returns: A classification-flavored signature_def. @@ -140,6 +149,10 @@ def classification_signature_def(examples, classes, scores): def predict_signature_def(inputs, outputs): """Creates prediction signature from given inputs and outputs. + This function produces signatures intended for use with the TensorFlow Serving + Predict API (tensorflow_serving/apis/prediction_service.proto). This API + imposes no constraints on the input and output types. + Args: inputs: dict of string to `Tensor`. outputs: dict of string to `Tensor`. diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py index 90afcc0a112dea884031fefe1504cce7a31c317a..355593eca5dd2f84419035958bfe8eea83e485b8 100644 --- a/tensorflow/python/summary/summary.py +++ b/tensorflow/python/summary/summary.py @@ -48,6 +48,7 @@ from tensorflow.core.util.event_pb2 import SessionLog from tensorflow.core.util.event_pb2 import TaggedRunMetadata # pylint: enable=unused-import +from tensorflow.python.eager import context as _context from tensorflow.python.framework import dtypes as _dtypes from tensorflow.python.framework import ops as _ops from tensorflow.python.ops import gen_logging_ops as _gen_logging_ops @@ -263,8 +264,20 @@ def merge(inputs, collections=None, name=None): Returns: A scalar `Tensor` of type `string`. The serialized `Summary` protocol buffer resulting from the merging. + + Raises: + RuntimeError: If called with eager mode enabled. + + @compatibility(eager) + Not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ # pylint: enable=line-too-long + if _context.in_eager_mode(): + raise RuntimeError( + 'Merging tf.summary.* ops is not compatible with eager execution. ' + 'Use tf.contrib.summary instead.') name = _summary_op_util.clean_tag(name) with _ops.name_scope(name, 'Merge', inputs): # pylint: disable=protected-access @@ -284,7 +297,19 @@ def merge_all(key=_ops.GraphKeys.SUMMARIES): If no summaries were collected, returns None. Otherwise returns a scalar `Tensor` of type `string` containing the serialized `Summary` protocol buffer resulting from the merging. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ + if _context.in_eager_mode(): + raise RuntimeError( + 'Merging tf.summary.* ops is not compatible with eager execution. ' + 'Use tf.contrib.summary instead.') summary_ops = _ops.get_collection(key) if not summary_ops: return None @@ -306,6 +331,11 @@ def get_summary_description(node_def): Raises: ValueError: if the node is not a summary op. + + @compatibility(eager) + Not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ if node_def.op != 'TensorSummary': @@ -317,7 +347,7 @@ def get_summary_description(node_def): _allowed_symbols = [ - 'Summary', 'SummaryDescription', 'Event', 'TaggedRunMetadata', 'SessionLog' + 'Summary', 'SummaryDescription', 'Event', 'TaggedRunMetadata', 'SessionLog', ] remove_undocumented(__name__, _allowed_symbols) diff --git a/tensorflow/python/summary/writer/writer.py b/tensorflow/python/summary/writer/writer.py index bd465335724629e7f4fe3a4f1c282f112c7d6796..12f120116f4439059f42c7212469ee835cc13ef4 100644 --- a/tensorflow/python/summary/writer/writer.py +++ b/tensorflow/python/summary/writer/writer.py @@ -25,6 +25,7 @@ from tensorflow.core.framework import graph_pb2 from tensorflow.core.framework import summary_pb2 from tensorflow.core.protobuf import meta_graph_pb2 from tensorflow.core.util import event_pb2 +from tensorflow.python.eager import context from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops from tensorflow.python.platform import gfile @@ -331,7 +332,20 @@ class FileWriter(SummaryToEventTransformer): graph_def: DEPRECATED: Use the `graph` argument instead. filename_suffix: A string. Every event file's name is suffixed with `suffix`. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + `FileWriter` is not compatible with eager execution. To write TensorBoard + summaries under eager execution, use `tf.contrib.summary` instead. + @end_compatbility """ + if context.in_eager_mode(): + raise RuntimeError( + "tf.summary.FileWriter is not compatible with eager execution. " + "Use tf.contrib.summary instead.") + event_writer = EventFileWriter(logdir, max_queue, flush_secs, filename_suffix) super(FileWriter, self).__init__(event_writer, graph, graph_def) diff --git a/tensorflow/python/tensorflow.i b/tensorflow/python/tensorflow.i index 9cef765bf3f4c6d339286253ed4b7b837c5689a8..d221dd523b2835d51e61487c22caee961ec28e5f 100644 --- a/tensorflow/python/tensorflow.i +++ b/tensorflow/python/tensorflow.i @@ -44,6 +44,8 @@ limitations under the License. %include "tensorflow/python/util/transform_graph.i" +%include "tensorflow/python/util/util.i" + %include "tensorflow/python/grappler/cluster.i" %include "tensorflow/python/grappler/item.i" %include "tensorflow/python/grappler/tf_optimizer.i" diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py index 96de9b921bcb2b492ab49866a93704b8b20ea75c..176d20bd60d3d042a5b7dc02387e3487b372b4a1 100644 --- a/tensorflow/python/training/adam_test.py +++ b/tensorflow/python/training/adam_test.py @@ -29,7 +29,6 @@ from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import resource_variable_ops -from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.training import adam @@ -153,7 +152,7 @@ class AdamOptimizerTest(test.TestCase): def doTestBasic(self, use_resource=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): - with variable_scope.variable_scope("%d" % i): + with self.test_session(graph=ops.Graph()): # Initialize variables for numpy implementation. m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0 var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) diff --git a/tensorflow/python/training/coordinator_test.py b/tensorflow/python/training/coordinator_test.py index 8f4cae6f06d683156f444b6174ba0564e04b7d7d..149d3eed414d53f46dcab403b7b4822ffa66e644 100644 --- a/tensorflow/python/training/coordinator_test.py +++ b/tensorflow/python/training/coordinator_test.py @@ -33,21 +33,26 @@ def StopOnEvent(coord, wait_for_stop, set_when_stopped): set_when_stopped.set() -def RaiseInN(coord, n_secs, ex, report_exception): +def RaiseOnEvent(coord, wait_for_stop, set_when_stopped, ex, report_exception): try: - time.sleep(n_secs) + wait_for_stop.wait() raise ex except RuntimeError as e: if report_exception: coord.request_stop(e) else: coord.request_stop(sys.exc_info()) + finally: + if set_when_stopped: + set_when_stopped.set() -def RaiseInNUsingContextHandler(coord, n_secs, ex): +def RaiseOnEventUsingContextHandler(coord, wait_for_stop, set_when_stopped, ex): with coord.stop_on_exception(): - time.sleep(n_secs) + wait_for_stop.wait() raise ex + if set_when_stopped: + set_when_stopped.set() def SleepABit(n_secs, coord=None): @@ -167,80 +172,113 @@ class CoordinatorTest(test.TestCase): def testJoinRaiseReportExcInfo(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() + ev_2 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("First"), False)), - threading.Thread(target=RaiseInN, - args=(coord, 0.05, RuntimeError("Too late"), False))] + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, ev_2, RuntimeError("First"), False)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_2, None, RuntimeError("Too late"), False))] for t in threads: t.start() + + ev_1.set() + with self.assertRaisesRegexp(RuntimeError, "First"): coord.join(threads) def testJoinRaiseReportException(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() + ev_2 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("First"), True)), - threading.Thread(target=RaiseInN, - args=(coord, 0.05, RuntimeError("Too late"), True))] + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, ev_2, RuntimeError("First"), True)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_2, None, RuntimeError("Too late"), True))] for t in threads: t.start() + + ev_1.set() with self.assertRaisesRegexp(RuntimeError, "First"): coord.join(threads) def testJoinIgnoresOutOfRange(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, - errors_impl.OutOfRangeError(None, None, "First"), - True)) + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, + errors_impl.OutOfRangeError(None, None, "First"), + True)) ] for t in threads: t.start() + + ev_1.set() coord.join(threads) def testJoinIgnoresMyExceptionType(self): coord = coordinator.Coordinator(clean_stop_exception_types=(ValueError,)) + ev_1 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, ValueError("Clean stop"), True)) + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, ValueError("Clean stop"), True)) ] for t in threads: t.start() + + ev_1.set() coord.join(threads) def testJoinRaiseReportExceptionUsingHandler(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() + ev_2 = threading.Event() threads = [ - threading.Thread(target=RaiseInNUsingContextHandler, - args=(coord, 0.01, RuntimeError("First"))), - threading.Thread(target=RaiseInNUsingContextHandler, - args=(coord, 0.05, RuntimeError("Too late")))] + threading.Thread( + target=RaiseOnEventUsingContextHandler, + args=(coord, ev_1, ev_2, RuntimeError("First"))), + threading.Thread( + target=RaiseOnEventUsingContextHandler, + args=(coord, ev_2, None, RuntimeError("Too late")))] for t in threads: t.start() + + ev_1.set() with self.assertRaisesRegexp(RuntimeError, "First"): coord.join(threads) def testClearStopClearsExceptionToo(self): coord = coordinator.Coordinator() + ev_1 = threading.Event() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("First"), True)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, RuntimeError("First"), True)), ] for t in threads: t.start() + with self.assertRaisesRegexp(RuntimeError, "First"): + ev_1.set() coord.join(threads) coord.clear_stop() threads = [ - threading.Thread(target=RaiseInN, - args=(coord, 0.01, RuntimeError("Second"), True)), + threading.Thread( + target=RaiseOnEvent, + args=(coord, ev_1, None, RuntimeError("Second"), True)), ] for t in threads: t.start() with self.assertRaisesRegexp(RuntimeError, "Second"): + ev_1.set() coord.join(threads) def testRequestStopRaisesIfJoined(self): diff --git a/tensorflow/python/training/evaluation.py b/tensorflow/python/training/evaluation.py index fdcb9c2e90a5adc7717d0bfc42bfb8b4a1c9a209..b36444a14c2ea98a973402664fd2055fcae655a7 100644 --- a/tensorflow/python/training/evaluation.py +++ b/tensorflow/python/training/evaluation.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function import time +import math from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -91,6 +92,9 @@ class _StopAfterNEvalsHook(session_run_hook.SessionRunHook): self._num_evals = num_evals self._evals_completed = None self._log_progress = log_progress + # Reduce logging frequency if there are 20 or more evaluations. + self._log_frequency = (1 if (num_evals is None or num_evals < 20) + else math.floor(num_evals / 10.)) def _set_evals_completed_tensor(self, updated_eval_step): self._evals_completed = updated_eval_step @@ -106,7 +110,9 @@ class _StopAfterNEvalsHook(session_run_hook.SessionRunHook): if self._num_evals is None: logging.info('Evaluation [%d]', evals_completed) else: - logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals) + if ((evals_completed % self._log_frequency) == 0 or + (self._num_evals == evals_completed)): + logging.info('Evaluation [%d/%d]', evals_completed, self._num_evals) if self._num_evals is not None and evals_completed >= self._num_evals: run_context.request_stop() diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py index 704017c244625e171a587789253fdb047cad0599..331a51e8bc848917967fed06632fe0d1c5bcad9c 100644 --- a/tensorflow/python/training/input.py +++ b/tensorflow/python/training/input.py @@ -27,12 +27,13 @@ import collections from six.moves import xrange # pylint: disable=redefined-builtin +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape -from tensorflow.python.framework import tensor_util +from tensorflow.python.layers import utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import data_flow_ops @@ -145,7 +146,18 @@ def input_producer(input_tensor, Raises: ValueError: If the shape of the input cannot be inferred from the arguments. + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError( + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") with ops.name_scope(name, "input_producer", [input_tensor]): input_tensor = ops.convert_to_tensor(input_tensor, name="input_tensor") element_shape = input_tensor.shape[1:].merge_with(element_shape) @@ -211,6 +223,11 @@ def string_input_producer(string_tensor, Raises: ValueError: If the string_tensor is a null Python list. At runtime, will fail with an assertion if string_tensor becomes a null tensor. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ not_null_err = "string_input_producer requires a non-null input tensor" if not isinstance(string_tensor, ops.Tensor) and not string_tensor: @@ -260,6 +277,11 @@ def range_input_producer(limit, num_epochs=None, shuffle=True, seed=None, Returns: A Queue with the output integers. A `QueueRunner` for the Queue is added to the current `Graph`'s `QUEUE_RUNNER` collection. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ with ops.name_scope(name, "input_producer", [limit]) as name: range_tensor = math_ops.range(limit) @@ -297,6 +319,11 @@ def slice_input_producer(tensor_list, num_epochs=None, shuffle=True, seed=None, Raises: ValueError: if `slice_input_producer` produces nothing from `tensor_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ with ops.name_scope(name, "input_producer", tensor_list): tensor_list = ops.convert_n_to_tensor_or_indexed_slices(tensor_list) @@ -413,22 +440,6 @@ def _as_original_type(original_tensors, tensor_list): return tensor_list -def _smart_cond(pred, if_true, if_false): - """A `tf.cond` that does nothing when the condition is static.""" - pred = ops.convert_to_tensor(pred) - static_pred = tensor_util.constant_value(pred) - if static_pred is not None: - if static_pred: - return if_true() - else: - return if_false() - else: - return control_flow_ops.cond( - pred, - if_true, - if_false) - - def _store_sparse_tensors(tensor_list, enqueue_many, keep_input, shared_map_ops=None): """Store SparseTensors for feeding into batch, etc. @@ -480,13 +491,13 @@ def _store_sparse_tensors(tensor_list, enqueue_many, keep_input, map_op_name = shared_map_op.name if shared_map_op else None def _maybe_store_sparse(t, map_op_name, keep_input): """Conditionally store a single sparse Tensor.""" - return _smart_cond( + return utils.smart_cond( keep_input, lambda: _store_sparse(t, shared_name=map_op_name), lambda: constant_op.constant(-1, dtypes.int64)) def _maybe_store_many_sparse(t, map_op_name, keep_input): """Conditionally store multiple sparse Tensors.""" - out_tensor = _smart_cond( + out_tensor = utils.smart_cond( keep_input, lambda: _store_many_sparse(t, shared_name=map_op_name), lambda: -1 * array_ops.ones(array_ops.shape(t)[0:1], dtypes.int64)) @@ -563,7 +574,23 @@ def _restore_sparse_tensors(stored_list, sparse_info_list): rank=(info.rank + 1).value) if info.sparse else s for (s, info) in zip(stored_list, sparse_info_list)] - return tensors if received_sequence else tensors[0] + has_st = any(isinstance(x, sparse_tensor.SparseTensor) for x in tensors) + if has_st: + t_values = [ + x.values if isinstance(x, sparse_tensor.SparseTensor) + else x + for x in tensors] + with_deps = lambda x: control_flow_ops.with_dependencies(t_values, x) + ensure_restore_tensors = [ + sparse_tensor.SparseTensor(indices=with_deps(x.indices), + values=with_deps(x.values), + dense_shape=with_deps(x.dense_shape)) + if isinstance(x, sparse_tensor.SparseTensor) + else with_deps(x) + for x in tensors] + else: + ensure_restore_tensors = tensors + return ensure_restore_tensors if received_sequence else tensors[0] def _validate(tensor_list): @@ -667,7 +694,7 @@ def _enqueue_join(queue, tensor_list_list, enqueue_many, keep_input): enqueue_ops = [enqueue_fn(_select_which_to_enqueue(x, keep_input)) for x in tensor_list_list] else: - enqueue_ops = [_smart_cond( + enqueue_ops = [utils.smart_cond( keep_input, lambda: enqueue_fn(tl), # pylint:disable=cell-var-from-loop control_flow_ops.no_op) for tl in tensor_list_list] @@ -684,7 +711,7 @@ def _enqueue(queue, tensor_list, threads, enqueue_many, keep_input): enqueue_ops = [ enqueue_fn(_select_which_to_enqueue(tensor_list, keep_input))] * threads else: - enqueue_ops = [_smart_cond( + enqueue_ops = [utils.smart_cond( keep_input, lambda: enqueue_fn(tensor_list), control_flow_ops.no_op)] * threads @@ -701,6 +728,11 @@ def _batch(tensors, batch_size, keep_input, num_threads=1, capacity=32, allow_smaller_final_batch=False, shared_name=None, name=None): """Helper function for `batch` and `maybe_batch`.""" + if context.in_eager_mode(): + raise ValueError( + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list = _as_tensor_list(tensors) with ops.name_scope(name, "batch", list(tensor_list) + [keep_input]) as name: tensor_list = _validate(tensor_list) @@ -734,6 +766,11 @@ def _batch_join(tensors_list, batch_size, keep_input, capacity=32, enqueue_many=False, shapes=None, dynamic_pad=False, allow_smaller_final_batch=False, shared_name=None, name=None): """Helper function for `batch_join` and `maybe_batch_join`.""" + if context.in_eager_mode(): + raise ValueError( + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list_list = _as_tensor_list_list(tensors_list) with ops.name_scope(name, "batch_join", _flatten(tensor_list_list) + [keep_input]) as name: @@ -764,6 +801,11 @@ def _shuffle_batch(tensors, batch_size, capacity, min_after_dequeue, shapes=None, allow_smaller_final_batch=False, shared_name=None, name=None): """Helper function for `shuffle_batch` and `maybe_shuffle_batch`.""" + if context.in_eager_mode(): + raise ValueError( + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list = _as_tensor_list(tensors) with ops.name_scope(name, "shuffle_batch", list(tensor_list) + [keep_input]) as name: @@ -804,6 +846,11 @@ def _shuffle_batch_join(tensors_list, batch_size, capacity, allow_smaller_final_batch=False, shared_name=None, name=None): """Helper function for `shuffle_batch_join` and `maybe_shuffle_batch_join`.""" + if context.in_eager_mode(): + raise ValueError( + "Input pipelines based on Queues are not supported when eager execution" + " is enabled. Please use tf.data to ingest data into your model" + " instead.") tensor_list_list = _as_tensor_list_list(tensors_list) with ops.name_scope(name, "shuffle_batch_join", _flatten(tensor_list_list) + [keep_input]) as name: @@ -912,6 +959,11 @@ def batch(tensors, batch_size, num_threads=1, capacity=32, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _batch( tensors, @@ -1065,6 +1117,11 @@ def batch_join(tensors_list, batch_size, capacity=32, enqueue_many=False, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensor_list_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _batch_join( tensors_list, @@ -1209,6 +1266,11 @@ def shuffle_batch(tensors, batch_size, capacity, min_after_dequeue, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch( tensors, @@ -1263,6 +1325,11 @@ def maybe_shuffle_batch(tensors, batch_size, capacity, min_after_dequeue, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch( tensors, @@ -1352,6 +1419,11 @@ def shuffle_batch_join(tensors_list, batch_size, capacity, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch_join( tensors_list, @@ -1406,6 +1478,11 @@ def maybe_shuffle_batch_join(tensors_list, batch_size, capacity, Raises: ValueError: If the `shapes` are not specified, and cannot be inferred from the elements of `tensors_list`. + + @compatibility(eager) + Input pipelines based on Queues are not supported when eager execution is + enabled. Please use the `tf.data` API to ingest data under eager execution. + @end_compatibility """ return _shuffle_batch_join( tensors_list, diff --git a/tensorflow/python/training/learning_rate_decay.py b/tensorflow/python/training/learning_rate_decay.py index bb9e26d8b474caa7a3f9912d211975532145ba46..802b930b0e391685b07802cbf6973b763e52d147 100644 --- a/tensorflow/python/training/learning_rate_decay.py +++ b/tensorflow/python/training/learning_rate_decay.py @@ -18,11 +18,14 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import math + from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import random_ops def exponential_decay(learning_rate, global_step, decay_steps, decay_rate, @@ -130,8 +133,12 @@ def piecewise_constant(x, boundaries, values, name=None): Raises: ValueError: if types of `x` and `boundaries` do not match, or types of all - `values` do not match. + `values` do not match or + the number of elements in the lists does not match. """ + if len(boundaries) != len(values) - 1: + raise ValueError( + "The length of boundaries should be 1 less than the length of values") with ops.name_scope(name, "PiecewiseConstant", [x, boundaries, values, name]) as name: x = ops.convert_to_tensor(x) @@ -158,14 +165,13 @@ def piecewise_constant(x, boundaries, values, name=None): raise ValueError( "Values must have elements all with the same dtype (%s vs %s)." % ( values[0].dtype.base_dtype, v.dtype.base_dtype)) - - pred_fn_pairs = {} - pred_fn_pairs[x <= boundaries[0]] = lambda: values[0] - pred_fn_pairs[x > boundaries[-1]] = lambda: values[-1] + pred_fn_pairs = [] + pred_fn_pairs.append((x <= boundaries[0], lambda: values[0])) + pred_fn_pairs.append((x > boundaries[-1], lambda: values[-1])) for low, high, v in zip(boundaries[:-1], boundaries[1:], values[1:-1]): # Need to bind v here; can do this with lambda v=v: ... pred = (x > low) & (x <= high) - pred_fn_pairs[pred] = lambda v=v: v + pred_fn_pairs.append((pred, lambda v=v: v)) # The default isn't needed here because our conditions are mutually # exclusive and exhaustive, but tf.case requires it. @@ -409,3 +415,226 @@ def inverse_time_decay(learning_rate, global_step, decay_steps, decay_rate, const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) denom = math_ops.add(const, math_ops.multiply(decay_rate, p)) return math_ops.div(learning_rate, denom, name=name) + + +def cosine_decay(learning_rate, global_step, decay_steps, name=None): + """Applies cosine decay to the learning rate. + + See [Loshchilov & Hutter, ICLR2016], SGDR: Stochastic Gradient Descent + with Warm Restarts. + + When training a model, it is often recommended to lower the learning rate as + the training progresses. This function applies a cosine decay function + to a provided initial learning rate. It requires a `global_step` value to + compute the decayed learning rate. You can just pass a TensorFlow variable + that you increment at each training step. + + The function returns the decayed learning rate. It is computed as: + ```python + global_step = min(global_step, decay_steps) + decayed = 0.5 * (1 + cos(pi * global_step / decay_steps)) + decayed_learning_rate = learning_rate * decayed + ``` + + Example usage: + ```python + decay_steps = 1000 + lr_decayed = cosine_decay(learning_rate, global_step, decay_steps) + ``` + + Args: + learning_rate: A scalar `float32` or `float64` Tensor or a Python number. + The initial learning rate. + global_step: A scalar `int32` or `int64` `Tensor` or a Python number. + Global step to use for the decay computation. + decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. + Number of steps to decay over. + name: String. Optional name of the operation. Defaults to 'CosineDecay'. + Returns: + A scalar `Tensor` of the same type as `learning_rate`. The decayed + learning rate. + Raises: + ValueError: if `global_step` is not supplied. + """ + if global_step is None: + raise ValueError("cosine decay requires global_step") + with ops.name_scope(name, "CosineDecay", + [learning_rate, global_step]) as name: + learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") + dtype = learning_rate.dtype + global_step = math_ops.cast(global_step, dtype) + decay_steps = math_ops.cast(decay_steps, dtype) + global_step = math_ops.minimum(global_step, decay_steps) + completed_fraction = global_step / decay_steps + cosine_decayed = 0.5 * ( + 1.0 + math_ops.cos(constant_op.constant(math.pi) * completed_fraction)) + + return math_ops.multiply(learning_rate, cosine_decayed) + + +def linear_cosine_decay(learning_rate, global_step, decay_steps, + num_periods=0.5, alpha=0.0, beta=0.001, + name=None): + """Applies linear cosine decay to the learning rate. + + See [Bello et al., ICML2017] Neural Optimizer Search with RL. + https://arxiv.org/abs/1709.07417 + + Note that linear cosine decay is more aggressive than cosine decay and + larger initial learning rates can typically be used. + + When training a model, it is often recommended to lower the learning rate as + the training progresses. This function applies a linear cosine decay function + to a provided initial learning rate. It requires a `global_step` value to + compute the decayed learning rate. You can just pass a TensorFlow variable + that you increment at each training step. + + The function returns the decayed learning rate. It is computed as: + ```python + global_step = min(global_step, decay_steps) + linear_decay = (decay_steps - global_step) / decay_steps) + cosine_decay = 0.5 * ( + 1 + cos(pi * 2 * num_periods * global_step / decay_steps)) + decayed = (alpha + linear_decay) * cosine_decay + beta + decayed_learning_rate = learning_rate * decayed + ``` + + Example usage: + ```python + decay_steps = 1000 + lr_decayed = linear_cosine_decay(learning_rate, global_step, decay_steps) + ``` + + Args: + learning_rate: A scalar `float32` or `float64` Tensor or a Python number. + The initial learning rate. + global_step: A scalar `int32` or `int64` `Tensor` or a Python number. + Global step to use for the decay computation. + decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. + Number of steps to decay over. + num_periods: Number of periods in the cosine part of the decay. + See computation above. + alpha: See computation above. + beta: See computation above. + name: String. Optional name of the operation. Defaults to + 'LinearCosineDecay'. + Returns: + A scalar `Tensor` of the same type as `learning_rate`. The decayed + learning rate. + Raises: + ValueError: if `global_step` is not supplied. + """ + if global_step is None: + raise ValueError("linear cosine decay requires global_step") + with ops.name_scope(name, "LinearCosineDecay", + [learning_rate, global_step]) as name: + learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") + dtype = learning_rate.dtype + global_step = math_ops.cast(global_step, dtype) + decay_steps = math_ops.cast(decay_steps, dtype) + num_periods = math_ops.cast(num_periods, dtype) + global_step = math_ops.minimum(global_step, decay_steps) + alpha = math_ops.cast(alpha, dtype) + beta = math_ops.cast(beta, dtype) + + linear_decayed = (decay_steps - global_step) / decay_steps + completed_fraction = global_step / decay_steps + fraction = 2.0 * num_periods * completed_fraction + cosine_decayed = 0.5 * ( + 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) + + linear_cosine_decayed = (alpha + linear_decayed) * cosine_decayed + beta + return math_ops.multiply(learning_rate, linear_cosine_decayed, name=name) + + +def noisy_linear_cosine_decay(learning_rate, global_step, decay_steps, + initial_variance=1.0, variance_decay=0.55, + num_periods=0.5, alpha=0.0, beta=0.001, + name=None): + """Applies noisy linear cosine decay to the learning rate. + + See [Bello et al., ICML2017] Neural Optimizer Search with RL. + https://arxiv.org/abs/1709.07417 + + Note that linear cosine decay is more aggressive than cosine decay and + larger initial learning rates can typically be used. + + When training a model, it is often recommended to lower the learning rate as + the training progresses. This function applies a noisy linear + cosine decay function to a provided initial learning rate. + It requires a `global_step` value to compute the decayed learning rate. + You can just pass a TensorFlow variable that you increment at each + training step. + + The function returns the decayed learning rate. It is computed as: + ```python + global_step = min(global_step, decay_steps) + linear_decay = (decay_steps - global_step) / decay_steps) + cosine_decay = 0.5 * ( + 1 + cos(pi * 2 * num_periods * global_step / decay_steps)) + decayed = (alpha + linear_decay + eps_t) * cosine_decay + beta + decayed_learning_rate = learning_rate * decayed + ``` + where eps_t is 0-centered gaussian noise with variance + initial_variance / (1 + global_step) ** variance_decay + + Example usage: + ```python + decay_steps = 1000 + lr_decayed = noisy_linear_cosine_decay( + learning_rate, global_step, decay_steps) + ``` + + Args: + learning_rate: A scalar `float32` or `float64` Tensor or a Python number. + The initial learning rate. + global_step: A scalar `int32` or `int64` `Tensor` or a Python number. + Global step to use for the decay computation. + decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. + Number of steps to decay over. + initial_variance: initial variance for the noise. See computation above. + variance_decay: decay for the noise's variance. See computation above. + num_periods: Number of periods in the cosine part of the decay. + See computation above. + alpha: See computation above. + beta: See computation above. + name: String. Optional name of the operation. Defaults to + 'NoisyLinearCosineDecay'. + Returns: + A scalar `Tensor` of the same type as `learning_rate`. The decayed + learning rate. + Raises: + ValueError: if `global_step` is not supplied. + """ + if global_step is None: + raise ValueError("noisy linear cosine decay requires global_step") + with ops.name_scope(name, "NoisyLinearCosineDecay", + [learning_rate, global_step]) as name: + learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") + dtype = learning_rate.dtype + global_step = math_ops.cast(global_step, dtype) + decay_steps = math_ops.cast(decay_steps, dtype) + global_step = math_ops.minimum(global_step, decay_steps) + initial_variance = math_ops.cast(initial_variance, dtype) + variance_decay = math_ops.cast(variance_decay, dtype) + num_periods = math_ops.cast(num_periods, dtype) + alpha = math_ops.cast(alpha, dtype) + beta = math_ops.cast(beta, dtype) + + linear_decayed = (decay_steps - global_step) / decay_steps + variance = initial_variance / ( + math_ops.pow(1.0 + global_step, variance_decay)) + std = math_ops.sqrt(variance) + noisy_linear_decayed = ( + linear_decayed + random_ops.random_normal( + linear_decayed.shape, stddev=std)) + + completed_fraction = global_step / decay_steps + fraction = 2.0 * num_periods * completed_fraction + cosine_decayed = 0.5 * ( + 1.0 + math_ops.cos(constant_op.constant(math.pi) * fraction)) + noisy_linear_cosine_decayed = ( + (alpha + noisy_linear_decayed) * cosine_decayed + beta) + + return math_ops.multiply( + learning_rate, noisy_linear_cosine_decayed, name=name) diff --git a/tensorflow/python/training/learning_rate_decay_test.py b/tensorflow/python/training/learning_rate_decay_test.py index 77da3099fe449cbe6e0ade734a9dbde5cb4c0452..ff41d80940a4b2f5d4c27f8691094422cd0cb18f 100644 --- a/tensorflow/python/training/learning_rate_decay_test.py +++ b/tensorflow/python/training/learning_rate_decay_test.py @@ -20,6 +20,7 @@ from __future__ import print_function import math +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import test_util from tensorflow.python.ops import gen_state_ops @@ -43,7 +44,7 @@ class LRDecayTest(test_util.TensorFlowTestCase): def testStaircase(self): with self.test_session(): step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_100 = state_ops.assign(step, 100) assign_1 = state_ops.assign(step, 1) assign_2 = state_ops.assign(step, 2) @@ -78,65 +79,63 @@ class LRDecayTest(test_util.TensorFlowTestCase): expected = .1 * 0.96 ** (100 // 3) self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + @test_util.run_in_graph_and_eager_modes() def testPiecewiseConstant(self): - with self.test_session(): - x = variables.Variable(-999) - assign_100 = x.assign(100) - assign_105 = x.assign(105) - assign_110 = x.assign(110) - assign_120 = x.assign(120) - assign_999 = x.assign(999) - pc = learning_rate_decay.piecewise_constant(x, [100, 110, 120], - [1.0, 0.1, 0.01, 0.001]) - - variables.global_variables_initializer().run() - self.assertAllClose(pc.eval(), 1.0, 1e-6) - assign_100.op.run() - self.assertAllClose(pc.eval(), 1.0, 1e-6) - assign_105.op.run() - self.assertAllClose(pc.eval(), 0.1, 1e-6) - assign_110.op.run() - self.assertAllClose(pc.eval(), 0.1, 1e-6) - assign_120.op.run() - self.assertAllClose(pc.eval(), 0.01, 1e-6) - assign_999.op.run() - self.assertAllClose(pc.eval(), 0.001, 1e-6) - + x = resource_variable_ops.ResourceVariable(-999) + def pc(): + return learning_rate_decay.piecewise_constant(x, [100, 110, 120], + [1.0, 0.1, 0.01, 0.001]) + + self.evaluate(variables.global_variables_initializer()) + + self.assertAllClose(self.evaluate(pc()), 1.0, 1e-6) + self.evaluate(x.assign(100)) + self.assertAllClose(self.evaluate(pc()), 1.0, 1e-6) + self.evaluate(x.assign(105)) + self.assertAllClose(self.evaluate(pc()), 0.1, 1e-6) + self.evaluate(x.assign(110)) + self.assertAllClose(self.evaluate(pc()), 0.1, 1e-6) + self.evaluate(x.assign(120)) + self.assertAllClose(self.evaluate(pc()), 0.01, 1e-6) + self.evaluate(x.assign(999)) + self.assertAllClose(self.evaluate(pc()), 0.001, 1e-6) + + @test_util.run_in_graph_and_eager_modes() def testPiecewiseConstantEdgeCases(self): - with self.test_session(): - x_int = variables.Variable(0, dtype=variables.dtypes.int32) - boundaries, values = [-1.0, 1.0], [1, 2, 3] - with self.assertRaises(ValueError): - learning_rate_decay.piecewise_constant(x_int, boundaries, values) + x_int = resource_variable_ops.ResourceVariable( + 0, dtype=variables.dtypes.int32) + boundaries, values = [-1.0, 1.0], [1, 2, 3] + with self.assertRaises(ValueError): + learning_rate_decay.piecewise_constant(x_int, boundaries, values) + x = resource_variable_ops.ResourceVariable(0.0) + boundaries, values = [-1.0, 1.0], [1.0, 2, 3] + with self.assertRaises(ValueError): + learning_rate_decay.piecewise_constant(x, boundaries, values) + + # Test that ref types are valid. + if context.in_graph_mode(): x = variables.Variable(0.0) - boundaries, values = [-1.0, 1.0], [1.0, 2, 3] - with self.assertRaises(ValueError): - learning_rate_decay.piecewise_constant(x, boundaries, values) - - # Test that ref types are valid. x_ref = x.op.outputs[0] # float32_ref tensor should be accepted boundaries, values = [1.0, 2.0], [1, 2, 3] learning_rate_decay.piecewise_constant(x_ref, boundaries, values) - # Test casting boundaries from int32 to int64. - x_int64 = variables.Variable(0, dtype=variables.dtypes.int64) - assign_1 = x_int64.assign(1) - assign_2 = x_int64.assign(2) - assign_3 = x_int64.assign(3) - assign_4 = x_int64.assign(4) - boundaries, values = [1, 2, 3], [0.4, 0.5, 0.6, 0.7] - pc = learning_rate_decay.piecewise_constant(x_int64, boundaries, values) - - variables.global_variables_initializer().run() - self.assertAllClose(pc.eval(), 0.4, 1e-6) - assign_1.op.run() - self.assertAllClose(pc.eval(), 0.4, 1e-6) - assign_2.op.run() - self.assertAllClose(pc.eval(), 0.5, 1e-6) - assign_3.op.run() - self.assertAllClose(pc.eval(), 0.6, 1e-6) - assign_4.op.run() - self.assertAllClose(pc.eval(), 0.7, 1e-6) + # Test casting boundaries from int32 to int64. + x_int64 = resource_variable_ops.ResourceVariable( + 0, dtype=variables.dtypes.int64) + boundaries, values = [1, 2, 3], [0.4, 0.5, 0.6, 0.7] + def pc(): + return learning_rate_decay.piecewise_constant(x_int64, boundaries, values) + + self.evaluate(variables.global_variables_initializer()) + self.assertAllClose(self.evaluate(pc()), 0.4, 1e-6) + self.evaluate(x_int64.assign(1)) + self.assertAllClose(self.evaluate(pc()), 0.4, 1e-6) + self.evaluate(x_int64.assign(2)) + self.assertAllClose(self.evaluate(pc()), 0.5, 1e-6) + self.evaluate(x_int64.assign(3)) + self.assertAllClose(self.evaluate(pc()), 0.6, 1e-6) + self.evaluate(x_int64.assign(4)) + self.assertAllClose(self.evaluate(pc()), 0.7, 1e-6) class LinearDecayTest(test_util.TensorFlowTestCase): @@ -245,6 +244,7 @@ class SqrtDecayTest(test_util.TensorFlowTestCase): expected = (lr - end_lr) * 0.25 ** power + end_lr self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + class PolynomialDecayTest(test_util.TensorFlowTestCase): def testBeginWithCycle(self): @@ -265,7 +265,7 @@ class ExponentialDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr, step, @@ -282,7 +282,7 @@ class ExponentialDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.natural_exp_decay(initial_lr, @@ -305,7 +305,7 @@ class InverseDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.inverse_time_decay(initial_lr, @@ -324,7 +324,7 @@ class InverseDecayTest(test_util.TensorFlowTestCase): k = 10 decay_rate = 0.96 step = gen_state_ops._variable(shape=[], dtype=dtypes.int32, - name="step", container="", shared_name="") + name="step", container="", shared_name="") assign_step = state_ops.assign(step, 0) increment_step = state_ops.assign_add(step, 1) decayed_lr = learning_rate_decay.inverse_time_decay(initial_lr, @@ -340,5 +340,98 @@ class InverseDecayTest(test_util.TensorFlowTestCase): increment_step.op.run() +class CosineDecayTest(test_util.TensorFlowTestCase): + + def np_cosine_decay(self, step, decay_steps): + step = min(step, decay_steps) + completed_fraction = step / decay_steps + return 0.5 * (1.0 + math.cos(math.pi * completed_fraction)) + + def testDecay(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + decayed_lr = learning_rate_decay.cosine_decay( + initial_lr, step, num_training_steps) + expected = self.np_cosine_decay(step, num_training_steps) + self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + + +class LinearCosineDecayTest(test_util.TensorFlowTestCase): + + def np_linear_cosine_decay(self, + step, + decay_steps, + alpha=0.0, + beta=0.001, + num_periods=0.5): + step = min(step, decay_steps) + linear_decayed = float(decay_steps - step) / decay_steps + fraction = 2.0 * num_periods * step / float(decay_steps) + cosine_decayed = 0.5 * (1.0 + math.cos(math.pi * fraction)) + return (alpha + linear_decayed) * cosine_decayed + beta + + def testDefaultDecay(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + decayed_lr = learning_rate_decay.linear_cosine_decay( + initial_lr, step, num_training_steps) + expected = self.np_linear_cosine_decay(step, num_training_steps) + self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + + def testNonDefaultDecay(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + decayed_lr = learning_rate_decay.linear_cosine_decay( + initial_lr, + step, + num_training_steps, + alpha=0.1, + beta=1e-4, + num_periods=5) + expected = self.np_linear_cosine_decay( + step, + num_training_steps, + alpha=0.1, + beta=1e-4, + num_periods=5) + self.assertAllClose(decayed_lr.eval(), expected, 1e-6) + + +class NoisyLinearCosineDecayTest(test_util.TensorFlowTestCase): + + def testDefaultNoisyLinearCosine(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + # No numerical check because of noise + decayed_lr = learning_rate_decay.noisy_linear_cosine_decay( + initial_lr, step, num_training_steps) + decayed_lr.eval() + + def testNonDefaultNoisyLinearCosine(self): + num_training_steps = 1000 + initial_lr = 1.0 + for step in range(0, 1500, 250): + with self.test_session(): + # No numerical check because of noise + decayed_lr = learning_rate_decay.noisy_linear_cosine_decay( + initial_lr, + step, + num_training_steps, + initial_variance=0.5, + variance_decay=0.1, + alpha=0.1, + beta=1e-4, + num_periods=5) + decayed_lr.eval() + + if __name__ == "__main__": googletest.main() diff --git a/tensorflow/python/training/momentum.py b/tensorflow/python/training/momentum.py index 7c00e219fda81c040e34dc2be37f2ebea9bd15c6..cf9530d87c46783b517884610b644b076bef6807 100644 --- a/tensorflow/python/training/momentum.py +++ b/tensorflow/python/training/momentum.py @@ -28,7 +28,7 @@ class MomentumOptimizer(optimizer.Optimizer): """Optimizer that implements the Momentum algorithm. Computes (if `use_nesterov = False`): - + ``` accumulation = momentum * accumulation + gradient variable -= learning_rate * accumulation @@ -58,6 +58,12 @@ class MomentumOptimizer(optimizer.Optimizer): variable(s) passed to the optimizer. Using Nesterov Momentum makes the variable(s) track the values called `theta_t + mu*v_t` in the paper. + @compatibility(eager) + When eager execution is enabled, learning_rate and momentum can each be a + callable that takes no arguments and returns the actual value to use. This + can be useful for changing these values across different invocations of + optimizer functions. + @end_compatibility """ super(MomentumOptimizer, self).__init__(use_locking, name) self._learning_rate = learning_rate @@ -69,10 +75,15 @@ class MomentumOptimizer(optimizer.Optimizer): self._zeros_slot(v, "momentum", self._name) def _prepare(self): - self._learning_rate_tensor = ops.convert_to_tensor(self._learning_rate, + learning_rate = self._learning_rate + if callable(learning_rate): + learning_rate = learning_rate() + self._learning_rate_tensor = ops.convert_to_tensor(learning_rate, name="learning_rate") - self._momentum_tensor = ops.convert_to_tensor(self._momentum, - name="momentum") + momentum = self._momentum + if callable(momentum): + momentum = momentum() + self._momentum_tensor = ops.convert_to_tensor(momentum, name="momentum") def _apply_dense(self, grad, var): mom = self.get_slot(var, "momentum") diff --git a/tensorflow/python/training/momentum_test.py b/tensorflow/python/training/momentum_test.py index d354ea443cf864053a4411296e050e4a740d64f3..3c8f472d6f9b3ae3ba62d348e7377a761409c29b 100644 --- a/tensorflow/python/training/momentum_test.py +++ b/tensorflow/python/training/momentum_test.py @@ -44,7 +44,7 @@ class MomentumOptimizerTest(test.TestCase): var = var - accum * lr * momentum return var, accum - def doTestBasic(self, use_resource=False): + def doTestBasic(self, use_resource=False, use_callable_params=False): for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): if use_resource: var0 = resource_variable_ops.ResourceVariable( @@ -56,8 +56,13 @@ class MomentumOptimizerTest(test.TestCase): var1 = variables.Variable([3.0, 4.0], dtype=dtype) grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + learning_rate = lambda: 2.0 + momentum = lambda: 0.9 + if not use_callable_params: + learning_rate = learning_rate() + momentum = momentum() mom_opt = momentum_lib.MomentumOptimizer( - learning_rate=2.0, momentum=0.9) + learning_rate=learning_rate, momentum=momentum) mom_update = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) @@ -125,6 +130,10 @@ class MomentumOptimizerTest(test.TestCase): def testResourceBasic(self): self.doTestBasic(use_resource=True) + def testBasicCallableParams(self): + with context.eager_mode(): + self.doTestBasic(use_resource=True, use_callable_params=True) + def testNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: with self.test_session(): diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py index e6162dd34b42e874bd896e04408d73ba3206ac69..dea62d27baf8ce8a9f2ae1dfcfe277b6927467a6 100644 --- a/tensorflow/python/training/monitored_session.py +++ b/tensorflow/python/training/monitored_session.py @@ -25,6 +25,7 @@ import sys import six from tensorflow.core.protobuf import config_pb2 +from tensorflow.python.estimator import util from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -348,8 +349,10 @@ def MonitoredTrainingSession(master='', # pylint: disable=invalid-name config=config) if checkpoint_dir: - all_hooks.append(basic_session_run_hooks.StepCounterHook( - output_dir=checkpoint_dir, every_n_steps=log_step_count_steps)) + if log_step_count_steps and log_step_count_steps > 0: + all_hooks.append( + basic_session_run_hooks.StepCounterHook( + output_dir=checkpoint_dir, every_n_steps=log_step_count_steps)) if (save_summaries_steps and save_summaries_steps > 0) or ( save_summaries_secs and save_summaries_secs > 0): @@ -493,6 +496,7 @@ class _MonitoredSession(object): self._sess = _RecoverableSession(self._coordinated_creator) else: self._sess = self._coordinated_creator.create_session() + self._stop_requested_in_step_fn = False @property def graph(self): @@ -520,10 +524,104 @@ class _MonitoredSession(object): options=options, run_metadata=run_metadata) + def run_step_fn(self, step_fn): + """Run ops using a step function. + + Args: + step_fn: A function or a method with a single argument of type + `StepContext`. The function may use methods of the argument to + perform computations with access to a raw session. + + The returned value of the `step_fn` will be returned from `run_step_fn`, + unless a stop is requested. In that case, the next `should_stop` call + will return True. + + Example usage: + ```python + with tf.Graph().as_default(): + c = tf.placeholder(dtypes.float32) + v = tf.add(c, 4.0) + w = tf.add(c, 0.5) + + def step_fn(step_context): + a = step_context.session.run(fetches=v, feed_dict={c: 0.5}) + if a <= 4.5: + step_context.request_stop() + return step_context.run_with_hooks(fetches=w, feed_dict={c: 0.1}) + + with tf.MonitoredSession() as session: + while not session.should_stop(): + a = session.run_step_fn(step_fn) + ``` + Hooks interact with the `run_with_hooks()` call inside the `step_fn` + as they do with a `MonitoredSession.run` call. + + Returns: + Returns the returned value of `step_fn`. + + Raises: + StopIteration: if `step_fn` has called `request_stop()`. It may be + caught by `with tf.MonitoredSession()` to close the session. + ValueError: if `step_fn` doesn't have a single argument called + `step_context`. It may also optionally have `self` for cases when it + belongs to an object. + """ + step_fn_arguments = util.fn_args(step_fn) + if step_fn_arguments != ('step_context',) and step_fn_arguments != ( + 'self', + 'step_context', + ): + raise ValueError( + '`step_fn` may either have one `step_context` argument, or' + ' `self` and `step_context` arguments if it\'s an instance' + ' method. Got {} instead.'.format(step_fn_arguments)) + + try: + return step_fn(_MonitoredSession.StepContext(self._tf_sess(), self.run)) + except StopIteration: + self._stop_requested_in_step_fn = True + raise + + class StepContext(object): + """Control flow instrument for the `step_fn` from `run_step_fn()`. + + Users of `step_fn` may perform `run()` calls without running hooks + by accessing the `session`. A `run()` call with hooks may be performed + using `run_with_hooks()`. Computation flow can be interrupted using + `request_stop()`. + """ + + def __init__(self, session, run_with_hooks_fn): + """Initializes the `step_context` argument for a `step_fn` invocation. + + Args: + session: An instance of `tf.Session`. + run_with_hooks_fn: A function for running fetches and hooks. + """ + self._session = session + self._run_with_hooks_fn = run_with_hooks_fn + + @property + def session(self): + return self._session + + def run_with_hooks(self, *args, **kwargs): + """Same as `MonitoredSession.run`. Accepts the same arguments.""" + return self._run_with_hooks_fn(*args, **kwargs) + + def request_stop(self): + """Exit the training loop by causing `should_stop()` to return `True`. + + Causes `step_fn` to exit by raising an exception. + + Raises: + StopIteration + """ + raise StopIteration('step_fn has requested the iterations to stop.') + def should_stop(self): - if self._sess: - return self._sess.should_stop() - return True + return (self._sess is None or self._sess.should_stop() or + self._stop_requested_in_step_fn) def close(self): self._close_internal() diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py index 84d262935aa88a693c3fa1f8afb85464d4872f3d..e729b79425fdc21f7c9d5be59bf9c14594534deb 100644 --- a/tensorflow/python/training/monitored_session_test.py +++ b/tensorflow/python/training/monitored_session_test.py @@ -33,10 +33,12 @@ from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import debug_pb2 from tensorflow.python.client import session as session_lib from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -1449,6 +1451,170 @@ class MonitoredSessionTest(test.TestCase): with monitored_session.MonitoredSession() as session: session.close() + def test_step_fn_example(self): + with ops.Graph().as_default(): + c = array_ops.placeholder(dtypes.float32) + v = array_ops.identity(c) + + def step_fn(step_context): + value = step_context.run_with_hooks(fetches=v, feed_dict={c: 3.2}) + return value + + with monitored_session.MonitoredSession() as session: + self.assertNear(3.2, session.run_step_fn(step_fn), 0.1) + + def test_step_function_stops(self): + with ops.Graph().as_default(): + + def step_fn(step_context): + step_context.request_stop() + + with monitored_session.MonitoredSession() as session: + self.assertEqual(None, session.run_step_fn(step_fn)) + self.assertTrue(session.should_stop()) + + def test_step_request_stop_without_a_with_block(self): + with ops.Graph().as_default(): + + def step_fn(step_context): + step_context.request_stop() + + session = monitored_session.MonitoredSession() + try: + self.assertEqual(None, session.run_step_fn(step_fn)) + except StopIteration: + pass + self.assertTrue(session.should_stop()) + + def test_step_request_stop_in_a_loop(self): + with ops.Graph().as_default(): + def step_fn(step_context): + step_context.request_stop() + + with monitored_session.MonitoredSession() as session: + while not session.should_stop(): + _ = session.run_step_fn(step_fn) + self.fail('An exception should be raised on the line above.') + + def test_step_request_stop_with_returning_a_type(self): + with ops.Graph().as_default(): + + def step_fn(step_context): + del step_context + return 'a type' + + with monitored_session.MonitoredSession() as session: + self.assertEqual('a type', session.run_step_fn(step_fn)) + + def test_step_with_extra_arguments(self): + with ops.Graph().as_default(): + + def step_fn(step_context, extra_foo): + del step_context, extra_foo + + with monitored_session.MonitoredSession() as session: + with self.assertRaisesRegexp( + ValueError, + '`step_fn` may either have one `step_context` argument'): + self.assertEqual(None, session.run_step_fn(step_fn)) + + def test_step_fn_belongs_to_a_class(self): + with ops.Graph().as_default(): + c = array_ops.placeholder(dtypes.float32) + v = array_ops.identity(c) + + class Model(object): + + def step_fn(self, step_context): + value = step_context.run_with_hooks(fetches=v, feed_dict={c: 3.2}) + return value + + with monitored_session.MonitoredSession() as session: + model = Model() + self.assertNear(3.2, session.run_step_fn(model.step_fn), 0.1) + + def test_step_fn_belongs_to_a_class_and_has_extra_methods(self): + with ops.Graph().as_default(): + + class Model(object): + + def step_fn(self, step_context, extra_foo): + del step_context, extra_foo + + with monitored_session.MonitoredSession() as session: + with self.assertRaisesRegexp( + ValueError, + '`step_fn` may either have one `step_context` argument'): + model = Model() + self.assertEqual(None, session.run_step_fn(model.step_fn)) + + def test_step_fn_with_hooks(self): + with ops.Graph().as_default(): + var = resource_variable_ops.ResourceVariable(0.0) + + # This test higlights the interaction of hooks with + # `Monitoredsession.run_step_fn`. The order of execution of operations + # below is: + # 0. stage_0 + # 1. stage_1_0 or stage_1_1 in an undefined order + # 2. stage_2 + + stage_0 = state_ops.assign_add(var, 0.3) + stage_1_0 = state_ops.assign_add(var, 0.7) + # The order of `stage_1_0` and `stage_1_1` is undefined by + # `MonitoredSession`, but we should be able to assert when both of them + # are complete. To obtain a consistent result of adding two different + # constants to `var`, we rely on a control dependency and + # `ResourceVariable`. Otherwise, it is possible that one of the + # additions overwites the result of the other addition. + with ops.control_dependencies([stage_1_0]): + stage_1_1 = state_ops.assign_add(var, 0.5) + stage_2 = state_ops.assign_add(var, 1.1) + + class Hook(session_run_hook.SessionRunHook): + + def __init__(self, testing): + self._testing = testing + + def before_run(self, run_context): + return session_run_hook.SessionRunArgs(fetches=stage_1_0) + + def after_run(self, run_context, run_values): + self._testing.assertNear(0.3 + 0.5 + 0.7, + run_context.session.run(var), 0.1) + self._testing.assertNear(0.3 + 0.5 + 0.7 + 1.1, + run_context.session.run(stage_2), 0.1) + + def step_fn(step_context): + self.assertNear(0.3, step_context.session.run(stage_0), 0.1) + return step_context.run_with_hooks(fetches=stage_1_1) + + with monitored_session.MonitoredSession(hooks=[Hook(self)]) as session: + self.assertEqual(0.3 + 0.5 + 0.7, session.run_step_fn(step_fn)) + + def test_step_fn_with_hooks_and_request_stop(self): + with ops.Graph().as_default(): + trace_the_hook = {'before_run': False, 'after_run': False} + + class Hook(session_run_hook.SessionRunHook): + + def before_run(self, run_context): + trace_the_hook['before_run'] = True + + def after_run(self, run_context, run_values): + trace_the_hook['after_run'] = True + + def step_fn(step_context): + step_context.request_stop() + + with monitored_session.MonitoredSession(hooks=[Hook()]) as session: + self.assertEqual(None, session.run_step_fn(step_fn)) + self.assertTrue(session.should_stop()) + # `step_context.request_stop()` in a step_fn interrupts the flow of + # running the hooks. + self.assertFalse(trace_the_hook['before_run']) + self.assertFalse(trace_the_hook['after_run']) + class SingularMonitoredSessionTest(test.TestCase): """Tests SingularMonitoredSession.""" diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py index 86ba8e2c8e471f453f9155778943aeacbad9941f..915214dbfaea022d6325c3cc122501687d3acf73 100644 --- a/tensorflow/python/training/optimizer.py +++ b/tensorflow/python/training/optimizer.py @@ -22,6 +22,7 @@ from __future__ import print_function import abc +from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops @@ -335,6 +336,17 @@ class Optimizer(object): Raises: ValueError: If some of the variables are not `Variable` objects. + + @compatibility(eager) + When eager execution is enabled, `loss` should be a Python function that + takes elements of `var_list` as arguments and computes the value to be + minimized. If `var_list` is None, `loss` should take no arguments. + Minimization (and gradient computation) is done with respect to the + elements of `var_list` if not None, else with respect to any trainable + variables created during the execution of the `loss` function. + `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and + `grad_loss` are ignored when eager execution is enabled. + @end_compatibility """ grads_and_vars = self.compute_gradients( loss, var_list=var_list, gate_gradients=gate_gradients, @@ -385,7 +397,37 @@ class Optimizer(object): Raises: TypeError: If `var_list` contains anything else than `Variable` objects. ValueError: If some arguments are invalid. + RuntimeError: If called with eager execution enabled and if `grad_loss` + is not `None` or `loss` is not callable. + + @compatibility(eager) + When eager execution is enabled, `loss` should be a Python function that + takes elements of `var_list` as arguments and computes the value to be + minimized. If `var_list` is None, `loss` should take no arguments. + Gradient computation is done with respect to the elements of `var_list` if + not None, else with respect to any trainable variables created during the + execution of the `loss` function. + `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and + `grad_loss` are ignored when eager execution is enabled. + @end_compatibility """ + if context.in_eager_mode(): + if grad_loss is not None: + raise RuntimeError( + "`grad_loss` argument to Optimizer.compute_gradients " + "not supported when eager execution is enabled.") + if not callable(loss): + raise RuntimeError( + "`loss` passed to Optimizer.compute_gradients should " + "be a function when eager execution is enabled.") + # TODO(agarwal): consider passing parameters to the `loss` function. + if var_list is None: + return backprop.implicit_grad(loss)() + else: + var_list = nest.flatten(var_list) + grads = backprop.gradients_function(loss)(*var_list) + grads_and_vars = list(zip(grads, var_list)) + return grads_and_vars if gate_gradients not in [Optimizer.GATE_NONE, Optimizer.GATE_OP, Optimizer.GATE_GRAPH]: raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, " @@ -489,11 +531,14 @@ class Optimizer(object): else: with ops.control_dependencies([self._finish(update_ops, "update")]): with ops.colocate_with(global_step): - apply_updates = state_ops.assign_add(global_step, 1, name=name).op - - train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) - if apply_updates not in train_op: - train_op.append(apply_updates) + apply_updates = state_ops.assign_add(global_step, 1, name=name) + + if context.in_graph_mode(): + if isinstance(apply_updates, ops.Tensor): + apply_updates = apply_updates.op + train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP) + if apply_updates not in train_op: + train_op.append(apply_updates) return apply_updates diff --git a/tensorflow/python/training/optimizer_test.py b/tensorflow/python/training/optimizer_test.py index c7eb9bc41236ee38655b40d9110a377afbca5a27..6bdae39073d48e0bd8b757a2d5145480e92d185f 100644 --- a/tensorflow/python/training/optimizer_test.py +++ b/tensorflow/python/training/optimizer_test.py @@ -18,12 +18,15 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.eager import context from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import clip_ops from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test @@ -32,26 +35,34 @@ from tensorflow.python.training import gradient_descent class OptimizerTest(test.TestCase): + @test_util.run_in_graph_and_eager_modes() def testBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a_%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b_%d' % i) + def loss(v0, v1): + return 5 * v0 + 3 * v1 + # Note that for eager execution, minimize expects a function instead of a + # Tensor. + cost = loss if context.in_eager_mode() else loss(var0, var1) + global_step = resource_variable_ops.ResourceVariable( + array_ops.zeros([], dtypes.int64), name='global_step_%d' % i) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - variables.global_variables_initializer().run() - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([-14., -13.], var0.eval()) - self.assertAllClose([-6., -5.], var1.eval()) + self.evaluate(variables.global_variables_initializer()) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + # Run 1 step of sgd through optimizer + opt_op = sgd_op.minimize(cost, global_step, [var0, var1]) + self.evaluate(opt_op) + # Validate updated params + self.assertAllClose([-14., -13.], self.evaluate(var0)) + self.assertAllClose([-6., -5.], self.evaluate(var1)) def testAggregationMethod(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: @@ -103,86 +114,112 @@ class OptimizerTest(test.TestCase): self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)], var1.eval()) + @test_util.run_in_graph_and_eager_modes() def testNoVariables(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype, trainable=False) - var1 = variables.Variable([3.0, 4.0], dtype=dtype, trainable=False) - cost = 5 * var0 + var1 - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, 'No variables'): - sgd_op.minimize(cost) + # pylint: disable=cell-var-from-loop + def loss(): + var0 = resource_variable_ops.ResourceVariable( + [1.0, 2.0], dtype=dtype, trainable=False, name='a') + var1 = resource_variable_ops.ResourceVariable( + [3.0, 4.0], dtype=dtype, trainable=False, name='b') + return 5 * var0 + var1 + # pylint: enable=cell-var-from-loop + cost = loss if context.in_eager_mode() else loss() + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, 'No.*variables'): + sgd_op.minimize(cost) + @test_util.run_in_graph_and_eager_modes() def testNoGradients(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, 'No gradients'): - # var1 has no gradient - sgd_op.minimize(cost, global_step, [var1]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b%d' % i) + # pylint: disable=cell-var-from-loop + def loss(_): + return 5 * var0 + # pylint: enable=cell-var-from-loop + cost = loss if context.in_eager_mode() else loss(var1) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, 'No gradients'): + # var1 has no gradient + sgd_op.minimize(cost, var_list=[var1]) + @test_util.run_in_graph_and_eager_modes() def testNoGradientsForAnyVariables_Minimize(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = constant_op.constant(5.0) - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, - 'No gradients provided for any variable'): - sgd_op.minimize(cost, global_step, [var0, var1]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a_%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b_%d' % i) + def loss(unused_v1, unused_v2): + return constant_op.constant(5.0) + cost = loss if context.in_eager_mode() else loss(var0, var1) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, + 'No gradients provided for any variable'): + sgd_op.minimize(cost, var_list=[var0, var1]) + @test_util.run_in_graph_and_eager_modes() def testNoGradientsForAnyVariables_ApplyGradients(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session(): - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - with self.assertRaisesRegexp(ValueError, - 'No gradients provided for any variable'): - sgd_op.apply_gradients([(None, var0), (None, var1)]) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a_%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b_%d' % i) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + with self.assertRaisesRegexp(ValueError, + 'No gradients provided for any variable'): + sgd_op.apply_gradients([(None, var0), (None, var1)]) + @test_util.run_in_graph_and_eager_modes() def testGradientsAsVariables(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - with self.test_session() as sess: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - cost = 5 * var0 + 3 * var1 - global_step = variables.Variable( - array_ops.zeros([], dtypes.int64), name='global_step') - sgd_op = gradient_descent.GradientDescentOptimizer(3.0) - grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1]) - # Convert gradients to tf.Variables - converted_grads = [ - variables.Variable(array_ops.zeros([2], dtype)) - for i in grads_and_vars - ] - convert_ops = [ - state_ops.assign(converted_grads[i], gv[0]) - for i, gv in enumerate(grads_and_vars) - ] - - converted_grads_and_vars = list(zip(converted_grads, [var0, var1])) - opt_op = sgd_op.apply_gradients(converted_grads_and_vars, global_step) + for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): + # Note that we name the variables uniquely here since the variables don't + # seem to be getting deleted at the end of the loop. + var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype, + name='a%d' % i) + var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype, + name='b%d' % i) + def loss(v0, v1): + return 5 * v0 + 3 * v1 + cost = loss if context.in_eager_mode() else loss(var0, var1) + sgd_op = gradient_descent.GradientDescentOptimizer(3.0) + grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1]) + # Convert gradients to tf.Variables + converted_grads = [ + resource_variable_ops.ResourceVariable(array_ops.zeros([2], dtype), + name='c_%d_%d' % (i, j)) + for j, gv in enumerate(grads_and_vars) + ] + convert_ops = [ + state_ops.assign(converted_grads[j], gv[0]) + for j, gv in enumerate(grads_and_vars) + ] - variables.global_variables_initializer().run() - # Run convert_ops to achieve the gradietns converting - sess.run(convert_ops) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], var0.eval()) - self.assertAllClose([3.0, 4.0], var1.eval()) - # Run 1 step of sgd through optimizer - opt_op.run() - # Validate updated params - self.assertAllClose([-14., -13.], var0.eval()) - self.assertAllClose([-6., -5.], var1.eval()) + self.evaluate(variables.global_variables_initializer()) + # Run convert_ops to achieve the gradietns converting + self.evaluate(convert_ops) + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + + # Run 1 step of sgd through optimizer + converted_grads_and_vars = list(zip(converted_grads, [var0, var1])) + opt_op = sgd_op.apply_gradients(converted_grads_and_vars) + self.evaluate(opt_op) + + # Validate updated params + self.assertAllClose([-14., -13.], self.evaluate(var0)) + self.assertAllClose([-6., -5.], self.evaluate(var1)) def testTrainOp(self): with self.test_session(): diff --git a/tensorflow/python/training/queue_runner_impl.py b/tensorflow/python/training/queue_runner_impl.py index 5abc6a2f5835bdad3d018bc67a84f8da97562c12..4e7c81d7b2913d71a23dcaa3751db2aaffdc67cf 100644 --- a/tensorflow/python/training/queue_runner_impl.py +++ b/tensorflow/python/training/queue_runner_impl.py @@ -23,6 +23,7 @@ import weakref from tensorflow.core.protobuf import queue_runner_pb2 from tensorflow.python.client import session +from tensorflow.python.eager import context from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.platform import tf_logging as logging @@ -43,6 +44,11 @@ class QueueRunner(object): and reporting exceptions, etc. The `QueueRunner`, combined with the `Coordinator`, helps handle these issues. + + @compatibility(eager) + QueueRunners are not compatible with eager execution. Instead, please + use `tf.data` to get data into your model. + @end_compatibility """ def __init__(self, queue=None, enqueue_ops=None, close_op=None, @@ -79,7 +85,13 @@ class QueueRunner(object): ValueError: If both `queue_runner_def` and `queue` are both specified. ValueError: If `queue` or `enqueue_ops` are not provided when not restoring from `queue_runner_def`. + RuntimeError: If eager execution is enabled. """ + if context.in_eager_mode(): + raise RuntimeError( + "QueueRunners are not supported when eager execution is enabled. " + "Instead, please use tf.data to get data into your model.") + if queue_runner_def: if queue or enqueue_ops: raise ValueError("queue_runner_def and queue are mutually exclusive.") @@ -414,7 +426,18 @@ def start_queue_runners(sess=None, coord=None, daemon=True, start=True, Returns: A list of threads. + + Raises: + RuntimeError: If called with eager execution enabled. + ValueError: If called without a default `tf.Session` registered. + + @compatibility(eager) + Not compatible with eager execution. To ingest data under eager execution, + use the `tf.data` API instead. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError("Queues are not compatible with eager execution.") if sess is None: sess = ops.get_default_session() if not sess: diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py index b1926f4eaf69a3e7e83629f962e2f6f6d170137b..60420eb86afb69cdd9caa92f07061f91c6631570 100644 --- a/tensorflow/python/training/saver.py +++ b/tensorflow/python/training/saver.py @@ -164,6 +164,7 @@ class BaseSaverBuilder(object): def __init__(self, var, slice_spec, name): self._var_device = var.device + self._var_shape = var.shape if isinstance(var, ops.Tensor): self.handle_op = var.op.inputs[0] tensor = var @@ -194,8 +195,8 @@ class BaseSaverBuilder(object): # Copy the restored tensor to the variable's device. with ops.device(self._var_device): restored_tensor = array_ops.identity(restored_tensor) - return resource_variable_ops.assign_variable_op( - self.handle_op, restored_tensor) + return resource_variable_ops.shape_safe_assign_variable_handle( + self.handle_op, self._var_shape, restored_tensor) def __init__(self, write_version=saver_pb2.SaverDef.V2): self._write_version = write_version @@ -557,7 +558,14 @@ class BaseSaverBuilder(object): if not isinstance(var, resource_variable_ops.ResourceVariable): raise ValueError("Can only save/restore ResourceVariable eager " "mode is enabled, type: %s." % type(var)) - names_to_saveables[var._shared_name] = var + set_var = names_to_saveables.setdefault(var._shared_name, var) + if set_var is not var: + raise ValueError( + ("Two different ResourceVariable objects with the same " + "shared_name '%s' were passed to the Saver. This likely means " + "that they were created in different Graphs or isolation " + "contexts, and may not be checkpointed together.") % ( + var._shared_name,)) # pylint: enable=protected-access return names_to_saveables @@ -1190,15 +1198,22 @@ class Saver(object): Raises: TypeError: If `var_list` is invalid. ValueError: If any of the keys or values in `var_list` are not unique. + RuntimeError: If eager execution is enabled and`var_list` does not specify + a list of varialbes to save. + + @compatibility(eager) + When eager execution is enabled, `var_list` must specify a `list` or `dict` + of variables to save. Otherwise, a `RuntimeError` will be raised. + @end_compatibility """ if defer_build and var_list: raise ValueError( "If `var_list` is provided then build cannot be deferred. " "Either set defer_build=False or var_list=None.") if context.in_eager_mode() and var_list is None: - raise ValueError( - "When eager execution is enabled, `var_list` must specify a list of " - "variables to save") + raise RuntimeError( + "When eager execution is enabled, `var_list` must specify a list or " + "dict of variables to save") self._var_list = var_list self._reshape = reshape self._sharded = sharded @@ -1223,7 +1238,7 @@ class Saver(object): def build(self): if context.in_eager_mode(): - raise ValueError("Use save/restore instead of build in eager mode.") + raise RuntimeError("Use save/restore instead of build in eager mode.") self._build(self._filename, build_save=True, build_restore=True) def _build_eager(self, checkpoint_path, build_save, build_restore): @@ -1495,18 +1510,17 @@ class Saver(object): It requires a session in which the graph was launched. The variables to save must also have been initialized. - The method returns the path of the newly created checkpoint file. This - path can be passed directly to a call to `restore()`. + The method returns the path prefix of the newly created checkpoint files. + This string can be passed directly to a call to `restore()`. Args: - sess: A Session to use to save the variables. None in eager mode. - save_path: String. Path to the checkpoint filename. If the saver is - `sharded`, this is the prefix of the sharded checkpoint filename. + sess: A Session to use to save the variables. + save_path: String. Prefix of filenames created for the checkpoint. global_step: If provided the global step number is appended to - `save_path` to create the checkpoint filename. The optional argument + `save_path` to create the checkpoint filenames. The optional argument can be a `Tensor`, a `Tensor` name or an integer. latest_filename: Optional name for the protocol buffer file that will - contains the list of most recent checkpoint filenames. That file, + contains the list of most recent checkpoints. That file, kept in the same directory as the checkpoint files, is automatically managed by the saver to keep track of recent checkpoints. Defaults to 'checkpoint'. @@ -1517,7 +1531,7 @@ class Saver(object): `CheckpointStateProto`. Returns: - A string: path at which the variables were saved. If the saver is + A string: path prefix used for the checkpoint files. If the saver is sharded, this string ends with: '-?????-of-nnnnn' where 'nnnnn' is the number of shards created. If the saver is empty, returns None. @@ -1794,11 +1808,19 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False, A None value is returned if no variables exist in the `MetaGraphDef` (i.e., there are no variables to restore). + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Exporting/importing meta graphs is not supported. No graph exists when eager + execution is enabled. + @end_compatibility """ # pylint: disable=g-doc-exception if context.in_eager_mode(): - raise ValueError("Exporting/importing meta graphs is not supported when " - "eager execution is enabled. No graph exists when eager " - "execution is enabled.") + raise RuntimeError("Exporting/importing meta graphs is not supported when " + "eager execution is enabled. No graph exists when eager " + "execution is enabled.") if not isinstance(meta_graph_or_file, meta_graph_pb2.MetaGraphDef): meta_graph_def = meta_graph.read_meta_graph_file(meta_graph_or_file) else: @@ -1864,11 +1886,17 @@ def export_meta_graph(filename=None, Raises: ValueError: When the `GraphDef` is larger than 2GB. + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Exporting/importing meta graphs is not supported. No graph exists when eager + execution is enabled. + @end_compatibility """ if context.in_eager_mode(): - raise ValueError("Exporting/importing meta graphs is not supported when " - "eager execution is enabled. No graph exists when eager " - "execution is enabled.") + raise RuntimeError("Exporting/importing meta graphs is not supported when " + "eager execution is enabled. No graph exists when eager " + "execution is enabled.") meta_graph_def, _ = meta_graph.export_scoped_meta_graph( filename=filename, meta_info_def=meta_info_def, diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py index a8eb8e5fcf9c7ae8ef84cd9f176368139764c4b8..744b17dd224297cbefedfe562ff106fe1200664f 100644 --- a/tensorflow/python/training/saver_test.py +++ b/tensorflow/python/training/saver_test.py @@ -110,32 +110,32 @@ class SaverTest(test.TestCase): # Start a second session. In that session the parameter nodes # have not been initialized either. with self.test_session(graph=ops_lib.Graph()) as sess: - v0_2 = variable_op(-1.0, name="v0") - v1_2 = variable_op(-1.0, name="v1") - v2_2 = saver_test_utils.CheckpointedOp(name="v2") + v0 = variable_op(-1.0, name="v0") + v1 = variable_op(-1.0, name="v1") + v2 = saver_test_utils.CheckpointedOp(name="v2") # Assert that the variables are not initialized. if context.in_graph_mode(): self.assertEqual( len(variables.report_uninitialized_variables().eval()), 2) - self.assertEqual(0, len(v2_2.keys().eval())) - self.assertEqual(0, len(v2_2.values().eval())) + self.assertEqual(0, len(v2.keys().eval())) + self.assertEqual(0, len(v2.values().eval())) # Restore the saved values in the parameter nodes. - save = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable}) + save = saver_module.Saver({"v0": v0, "v1": v1, "v2": v2.saveable}) save.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, self.evaluate(v0_2)) - self.assertEqual(20.0, self.evaluate(v1_2)) - self.assertEqual(b"k1", self.evaluate(v2_2.keys())) - self.assertEqual(30.0, self.evaluate(v2_2.values())) + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) + self.assertEqual(b"k1", self.evaluate(v2.keys())) + self.assertEqual(30.0, self.evaluate(v2.values())) # Build another graph with 2 nodes, initialized # differently, and a Restore node for them. with self.test_session(graph=ops_lib.Graph()) as sess: - v0_3 = variable_op(1000.0, name="v0") - v1_3 = variable_op(2000.0, name="v1") - v2_3 = saver_test_utils.CheckpointedOp(name="v2") - v2_init = v2_3.insert("k1000", 3000.0) + v0_2 = variable_op(1000.0, name="v0") + v1_2 = variable_op(2000.0, name="v1") + v2_2 = saver_test_utils.CheckpointedOp(name="v2") + v2_init = v2_2.insert("k1000", 3000.0) # Check that the parameter nodes have been initialized. if context.in_graph_mode(): @@ -143,19 +143,19 @@ class SaverTest(test.TestCase): self.evaluate(init_all_op) # TODO(xpan): Why _mutable_hash_table_v2 doesn't create empty # table as it claims in eager mode? - self.assertEqual(b"k1000", self.evaluate(v2_3.keys())) - self.assertEqual(3000.0, self.evaluate(v2_3.values())) - self.assertEqual(1000.0, self.evaluate(v0_3)) - self.assertEqual(2000.0, self.evaluate(v1_3)) + self.assertEqual(b"k1000", self.evaluate(v2_2.keys())) + self.assertEqual(3000.0, self.evaluate(v2_2.values())) + self.assertEqual(1000.0, self.evaluate(v0_2)) + self.assertEqual(2000.0, self.evaluate(v1_2)) # Restore the values saved earlier in the parameter nodes. - save2 = saver_module.Saver({"v0": v0_3, "v1": v1_3, "v2": v2_3.saveable}) + save2 = saver_module.Saver({"v0": v0_2, "v1": v1_2, "v2": v2_2.saveable}) save2.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, self.evaluate(v0_3)) - self.assertEqual(20.0, self.evaluate(v1_3)) - self.assertEqual(b"k1", self.evaluate(v2_3.keys())) - self.assertEqual(30.0, self.evaluate(v2_3.values())) + self.assertEqual(10.0, self.evaluate(v0_2)) + self.assertEqual(20.0, self.evaluate(v1_2)) + self.assertEqual(b"k1", self.evaluate(v2_2.keys())) + self.assertEqual(30.0, self.evaluate(v2_2.values())) def testBasic(self): self.basicSaveRestore(variables.Variable) @@ -233,7 +233,8 @@ class SaverTest(test.TestCase): def testResourceSaveRestoreCachingDevice(self): save_path = os.path.join(self.get_temp_dir(), "resource_cache") with self.test_session(graph=ops_lib.Graph()) as sess: - v = resource_variable_ops.ResourceVariable([1], caching_device="/cpu:0") + v = resource_variable_ops.ResourceVariable([1], caching_device="/cpu:0", + name="v") if context.in_graph_mode(): self.evaluate(variables.global_variables_initializer()) else: @@ -479,18 +480,18 @@ class SaverTest(test.TestCase): self.assertEqual(30.0, v2_2.values().eval()) def _SaveAndLoad(self, var_name, var_value, other_value, save_path): - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: var = resource_variable_ops.ResourceVariable(var_value, name=var_name) save = saver_module.Saver({var_name: var}) if context.in_graph_mode(): self.evaluate(var.initializer) val = save.save(sess, save_path) self.assertEqual(save_path, val) - with self.test_session() as sess: - var2 = resource_variable_ops.ResourceVariable(other_value, name=var_name) - save = saver_module.Saver({var_name: var2}) + with self.test_session(graph=ops_lib.Graph()) as sess: + var = resource_variable_ops.ResourceVariable(other_value, name=var_name) + save = saver_module.Saver({var_name: var}) save.restore(sess, save_path) - self.assertAllClose(var_value, self.evaluate(var2)) + self.assertAllClose(var_value, self.evaluate(var)) def testCacheRereadsFile(self): save_path = os.path.join(self.get_temp_dir(), "cache_rereads") @@ -618,8 +619,8 @@ class SaverTest(test.TestCase): global_step_int = 5 # Save and reload one Variable named "var0". self._SaveAndLoad("var0", 0.0, 1.0, save_path) - for i, use_tensor in enumerate([True, False]): - with variable_scope.variable_scope("%d" % i): + for use_tensor in [True, False]: + with self.test_session(graph=ops_lib.Graph()): var = resource_variable_ops.ResourceVariable(1.0, name="var0") save = saver_module.Saver( { @@ -1298,20 +1299,20 @@ class KeepCheckpointEveryNHoursTest(test.TestCase): class SaveRestoreWithVariableNameMap(test.TestCase): - def testNonReshape(self): + def _testNonReshape(self, variable_op): save_path = os.path.join(self.get_temp_dir(), "non_reshape") - with self.test_session() as sess: + with self.test_session(graph=ops_lib.Graph()) as sess: # Build a graph with 2 parameter nodes, and Save and # Restore nodes for them. - v0 = variables.Variable(10.0, name="v0") - v1 = variables.Variable(20.0, name="v1") + v0 = variable_op(10.0, name="v0") + v1 = variable_op(20.0, name="v1") save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1}) - variables.global_variables_initializer().run() + self.evaluate(variables.global_variables_initializer()) # Check that the parameter nodes have been initialized. - self.assertEqual(10.0, v0.eval()) - self.assertEqual(20.0, v1.eval()) + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) # Save the initialized values in the file at "save_path" # Use a variable name map to set the saved tensor names @@ -1326,40 +1327,50 @@ class SaveRestoreWithVariableNameMap(test.TestCase): # Verify that the mapped names are present in the Saved file and can be # Restored using remapped names. - with self.test_session() as sess: - v0 = variables.Variable(-1.0, name="v0") - v1 = variables.Variable(-1.0, name="v1") + with self.test_session(graph=ops_lib.Graph()) as sess: + v0 = variable_op(-1.0, name="v0") + v1 = variable_op(-1.0, name="v1") - with self.assertRaisesOpError("uninitialized value v0"): - sess.run(v0) - with self.assertRaisesOpError("uninitialized value v1"): - sess.run(v1) + if context.in_graph_mode(): + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v0) + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v1) save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1}) save.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, v0.eval()) - self.assertEqual(20.0, v1.eval()) + if context.in_graph_mode(): + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) # Add a prefix to the node names in the current graph and Restore using # remapped names. - with self.test_session() as sess: - v0 = variables.Variable(-1.0, name="restore_prefix/v0") - v1 = variables.Variable(-1.0, name="restore_prefix/v1") + with self.test_session(graph=ops_lib.Graph()) as sess: + v0 = variable_op(-1.0, name="restore_prefix/v0") + v1 = variable_op(-1.0, name="restore_prefix/v1") - with self.assertRaisesOpError("uninitialized value restore_prefix/v0"): - sess.run(v0) - with self.assertRaisesOpError("uninitialized value restore_prefix/v1"): - sess.run(v1) + if context.in_graph_mode(): + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v0) + with self.assertRaisesOpError("uninitialized"): + self.evaluate(v1) # Restore the saved values in the parameter nodes. save = saver_module.Saver({"save_prefix/v0": v0, "save_prefix/v1": v1}) save.restore(sess, save_path) # Check that the parameter nodes have been restored. - self.assertEqual(10.0, v0.eval()) - self.assertEqual(20.0, v1.eval()) + self.assertEqual(10.0, self.evaluate(v0)) + self.assertEqual(20.0, self.evaluate(v1)) + + @test_util.run_in_graph_and_eager_modes() + def testNonReshapeResourceVariable(self): + self._testNonReshape(resource_variable_ops.ResourceVariable) + + def testNonReshapeVariable(self): + self._testNonReshape(variables.Variable) class LatestCheckpointWithRelativePaths(test.TestCase): diff --git a/tensorflow/python/training/supervisor.py b/tensorflow/python/training/supervisor.py index cfdd03dc1505204b40e06e0c20f454397890eec8..a634a842b67033d5fde6bf8cf819f681e892a247 100644 --- a/tensorflow/python/training/supervisor.py +++ b/tensorflow/python/training/supervisor.py @@ -23,6 +23,7 @@ import time from tensorflow.core.framework.summary_pb2 import Summary from tensorflow.core.util.event_pb2 import SessionLog +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import meta_graph from tensorflow.python.framework import ops @@ -288,7 +289,16 @@ class Supervisor(object): Returns: A `Supervisor`. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + `Supervisor`s are not supported when eager execution is enabled. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError("Supervisors are compatible with eager execution.") # Set default values of arguments. if graph is None: graph = ops.get_default_graph() @@ -735,7 +745,17 @@ class Supervisor(object): Returns: The list of threads started for the `QueueRunners`. + + Raises: + RuntimeError: If called with eager execution enabled. + + @compatibility(eager) + Queues are not compatible with eager execution. To ingest data when eager + execution is enabled, use the `tf.data` API. + @end_compatibility """ + if context.in_eager_mode(): + raise RuntimeError("Queues are not compatible with eager execution.") if queue_runners is None: queue_runners = self._graph.get_collection(ops.GraphKeys.QUEUE_RUNNERS) threads = [] @@ -768,7 +788,10 @@ class Supervisor(object): looper.start() return looper - def stop(self, threads=None, close_summary_writer=True): + def stop(self, + threads=None, + close_summary_writer=True, + ignore_live_threads=False): """Stop the services and the coordinator. This does not close the session. @@ -782,14 +805,19 @@ class Supervisor(object): close_summary_writer: Whether to close the `summary_writer`. Defaults to `True` if the summary writer was created by the supervisor, `False` otherwise. + ignore_live_threads: If `True` ignores threads that remain running after + a grace period when joining threads via the coordinator, instead of + raising a RuntimeError. """ self._coord.request_stop() try: # coord.join() re-raises the first reported exception; the "finally" # block ensures that we clean up whether or not an exception was # reported. - self._coord.join(threads, - stop_grace_period_secs=self._stop_grace_secs) + self._coord.join( + threads, + stop_grace_period_secs=self._stop_grace_secs, + ignore_live_threads=ignore_live_threads) finally: # Close the writer last, in case one of the running threads was using it. if close_summary_writer and self._summary_writer: diff --git a/tensorflow/python/training/training.py b/tensorflow/python/training/training.py index 741dddc991ff570b39de747069ea6898363d5023..fa02ad84cce3ccaa391571df3a2de4b65b255c84 100644 --- a/tensorflow/python/training/training.py +++ b/tensorflow/python/training/training.py @@ -37,6 +37,9 @@ See the @{$python/train} guide. @@clip_by_average_norm @@clip_by_global_norm @@global_norm +@@cosine_decay +@@linear_cosine_decay +@@noisy_linear_cosine_decay @@exponential_decay @@inverse_time_decay @@natural_exp_decay diff --git a/tensorflow/python/training/training_util.py b/tensorflow/python/training/training_util.py index bdd4ca734eb260ca246cd00c168ba42fe3538f0e..89a9e129328fe38da2ce497a7f26dc11446ea032 100644 --- a/tensorflow/python/training/training_util.py +++ b/tensorflow/python/training/training_util.py @@ -119,13 +119,24 @@ def create_global_step(graph=None): raise ValueError('"global_step" already exists.') # Create in proper graph and base name_scope. with graph.as_default() as g, g.name_scope(None): + if context.in_eager_mode(): + with ops.device('cpu:0'): + return variable_scope.get_variable( + ops.GraphKeys.GLOBAL_STEP, + shape=[], + dtype=dtypes.int64, + initializer=init_ops.zeros_initializer(), + trainable=False, + collections=[ops.GraphKeys.GLOBAL_VARIABLES, + ops.GraphKeys.GLOBAL_STEP]) return variable_scope.get_variable( ops.GraphKeys.GLOBAL_STEP, shape=[], dtype=dtypes.int64, initializer=init_ops.zeros_initializer(), trainable=False, - collections=[ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.GLOBAL_STEP]) + collections=[ops.GraphKeys.GLOBAL_VARIABLES, + ops.GraphKeys.GLOBAL_STEP]) def get_or_create_global_step(graph=None): diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py index d57140da75038f6559b1a51a3d9c2c6894c608af..dd6acee3c7537827808ec98561f3ea7fd80910d0 100644 --- a/tensorflow/python/util/nest.py +++ b/tensorflow/python/util/nest.py @@ -35,7 +35,7 @@ import collections as _collections import six as _six -from tensorflow.python.platform import tf_logging as _tf_logging +from tensorflow.python import pywrap_tensorflow as _pywrap_tensorflow from tensorflow.python.util.all_util import remove_undocumented @@ -91,26 +91,6 @@ def _yield_value(iterable): yield value -def _yield_flat_nest(nest): - for n in _yield_value(nest): - if is_sequence(n): - for ni in _yield_flat_nest(n): - yield ni - else: - yield n - - -# Used by `_warn_once` to remember which warning messages have been given. -_ALREADY_WARNED = {} - - -def _warn_once(message): - """Logs a warning message, once per unique string.""" - if message not in _ALREADY_WARNED: - _ALREADY_WARNED[message] = True - _tf_logging.warning(message) - - def is_sequence(seq): """Returns a true if its input is a collections.Sequence (except strings). @@ -121,13 +101,7 @@ def is_sequence(seq): True if the sequence is a not a string and is a collections.Sequence or a dict. """ - if isinstance(seq, dict): - return True - if isinstance(seq, set): - _warn_once("Sets are not currently considered sequences, but this may " - "change in the future, so consider avoiding using them.") - return (isinstance(seq, _collections.Sequence) - and not isinstance(seq, _six.string_types)) + return _pywrap_tensorflow.IsSequence(seq) def flatten(nest): @@ -145,6 +119,9 @@ def flatten(nest): a correponding plain dict, or vice-versa. Dictionaries with non-sortable keys cannot be flattened. + Users must not modify any collections used in `nest` while this function is + running. + Args: nest: an arbitrarily nested structure or a scalar object. Note, numpy arrays are considered scalars. @@ -155,10 +132,7 @@ def flatten(nest): Raises: TypeError: The nest is or contains a dict with non-sortable keys. """ - if is_sequence(nest): - return list(_yield_flat_nest(nest)) - else: - return [nest] + return _pywrap_tensorflow.Flatten(nest) def _recursive_assert_same_structure(nest1, nest2, check_types): @@ -692,6 +666,9 @@ def get_traverse_shallow_structure(traverse_fn, structure): return _sequence_like(structure, level_traverse) +_pywrap_tensorflow.RegisterSequenceClass(_collections.Sequence) + + _allowed_symbols = [ "assert_same_structure", "is_sequence", diff --git a/tensorflow/python/util/tf_should_use.py b/tensorflow/python/util/tf_should_use.py index d9b2e6fcd799db019adf40c717efc09845aa216f..99081cb29470900992f4583445817521e8dd2553 100644 --- a/tensorflow/python/util/tf_should_use.py +++ b/tensorflow/python/util/tf_should_use.py @@ -44,6 +44,11 @@ def _add_should_use_warning(x, fatal_error=False): if x is None: # special corner case where x is None return x + # TODO(apassos) we don't have an easier way to check because importing context + # or ops here would create a BUILD dependency cycle. + if type(x).__name__ == 'EagerTensor': + return x + def override_method(method): def fn(self, *args, **kwargs): return method(self, *args, **kwargs) diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc new file mode 100644 index 0000000000000000000000000000000000000000..c3d7611ad43b05f510481925fbfe1f930cf95ff8 --- /dev/null +++ b/tensorflow/python/util/util.cc @@ -0,0 +1,127 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/python/util/util.h" + +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { +namespace swig { + +namespace { + +// Type object for collections.Sequence. This is set by RegisterSequenceClass. +PyObject* CollectionsSequenceType = nullptr; + +bool WarnedThatSetIsNotSequence = false; + +// Returns 1 if `o` is considered a sequence for the purposes of Flatten(). +// Returns 0 otherwise. +// Returns -1 if an error occured. +int IsSequenceHelper(PyObject* o) { + if (PyDict_Check(o)) return true; + if (PySet_Check(o) && !WarnedThatSetIsNotSequence) { + LOG(WARNING) << "Sets are not currently considered sequences, " + "but this may change in the future, " + "so consider avoiding using them."; + WarnedThatSetIsNotSequence = true; + } + if (CollectionsSequenceType == nullptr) { + PyErr_SetString( + PyExc_RuntimeError, + tensorflow::strings::StrCat( + "collections.Sequence type has not been set. " + "Please call RegisterSequenceClass before using this module") + .c_str()); + return -1; + } + int is_instance = PyObject_IsInstance(o, CollectionsSequenceType); + if (is_instance == -1) return -1; + return static_cast(is_instance != 0 && !PyBytes_Check(o) && +#if PY_MAJOR_VERSION < 3 + !PyString_Check(o) && +#endif + !PyUnicode_Check(o)); +} + +bool FlattenHelper(PyObject* nested, PyObject* list) { + // if nested is not a sequence, append itself and exit + int is_seq = IsSequenceHelper(nested); + if (is_seq == -1) return false; + if (!is_seq) { + return PyList_Append(list, nested) != -1; + } + + // if nested if dictionary, sort it by key and recurse on each value + if (PyDict_Check(nested)) { + PyObject* keys = PyDict_Keys(nested); + if (PyList_Sort(keys) == -1) return false; + Py_ssize_t size = PyList_Size(keys); + for (Py_ssize_t i = 0; i < size; ++i) { + // We know that key and val will not be deleted because nested owns + // a reference to them and callers of flatten must not modify nested + // while the method is running. + PyObject* key = PyList_GET_ITEM(keys, i); + PyObject* val = PyDict_GetItem(nested, key); + if (Py_EnterRecursiveCall(" in Flatten")) { + Py_DECREF(keys); + return false; + } + FlattenHelper(val, list); + Py_LeaveRecursiveCall(); + } + Py_DECREF(keys); + return true; + } + + // iterate and recurse + PyObject* item; + PyObject* iterator = PyObject_GetIter(nested); + while ((item = PyIter_Next(iterator)) != nullptr) { + FlattenHelper(item, list); + Py_DECREF(item); + } + Py_DECREF(iterator); + return true; +} + +} // anonymous namespace + +void RegisterSequenceClass(PyObject* sequence_class) { + if (!PyType_Check(sequence_class)) { + PyErr_SetString( + PyExc_TypeError, + tensorflow::strings::StrCat( + "Expecting a class definition for `collections.Sequence`. Got ", + Py_TYPE(sequence_class)->tp_name) + .c_str()); + return; + } + CollectionsSequenceType = sequence_class; +} + +bool IsSequence(PyObject* o) { return IsSequenceHelper(o) == 1; } + +PyObject* Flatten(PyObject* nested) { + PyObject* list = PyList_New(0); + if (FlattenHelper(nested, list)) { + return list; + } else { + Py_DECREF(list); + return nullptr; + } +} +} // namespace swig +} // namespace tensorflow diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h new file mode 100644 index 0000000000000000000000000000000000000000..493d26b497d714b318a345c96462d2d01de789c9 --- /dev/null +++ b/tensorflow/python/util/util.h @@ -0,0 +1,74 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Functions for getting information about kernels registered in the binary. +#ifndef THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_ +#define THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_ + +#include + +namespace tensorflow { +namespace swig { + +// Implements the same interface as tensorflow.util.nest.is_sequence +// Returns a true if its input is a collections.Sequence (except strings). +// +// Args: +// seq: an input sequence. +// +// Returns: +// True if the sequence is a not a string and is a collections.Sequence or a +// dict. +bool IsSequence(PyObject* o); + +// Implements the same interface as tensorflow.util.nest.flatten +// +// Returns a flat list from a given nested structure. +// +// If `nest` is not a sequence, tuple, or dict, then returns a single-element +// list: `[nest]`. +// +// In the case of dict instances, the sequence consists of the values, sorted by +// key to ensure deterministic behavior. This is true also for `OrderedDict` +// instances: their sequence order is ignored, the sorting order of keys is +// used instead. The same convention is followed in `pack_sequence_as`. This +// correctly repacks dicts and `OrderedDict`s after they have been flattened, +// and also allows flattening an `OrderedDict` and then repacking it back using +// a correponding plain dict, or vice-versa. +// Dictionaries with non-sortable keys cannot be flattened. +// +// Args: +// nest: an arbitrarily nested structure or a scalar object. Note, numpy +// arrays are considered scalars. +// +// Returns: +// A Python list, the flattened version of the input. +// On error, returns nullptr +// +// Raises: +// TypeError: The nest is or contains a dict with non-sortable keys. +PyObject* Flatten(PyObject* nested); + +// RegisterSequenceClass is used to pass PyTypeObject for collections.Sequence +// (which is defined in python) into the C++ world. +// Alternative approach could be to import the collections modules and retrieve +// the type from the module. This approach also requires some trigger from +// Python so that we know that Python interpreter had been initialzied. +void RegisterSequenceClass(PyObject* sequence_class); + +} // namespace swig +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_PYTHON_UTIL_UTIL_H_ diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i new file mode 100644 index 0000000000000000000000000000000000000000..d69084fc0091ac79cf3f5cf3d70af419cf78f936 --- /dev/null +++ b/tensorflow/python/util/util.i @@ -0,0 +1,42 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +%include "tensorflow/python/platform/base.i" + +%{ +#include "tensorflow/python/util/util.h" +%} + +%ignoreall + +%unignore tensorflow; +%unignore tensorflow::swig; +// The %exception block defined in tf_session.i releases the Python GIL for +// the length of each wrapped method. This file is included in tensorflow.i +// after tf_session.i and inherits this definition. We disable this behavior +// for functions in this module because they use python methods that need GIL. +// TODO(iga): Find a way not to leak such definitions across files. + +%unignore tensorflow::swig::RegisterSequenceClass; +%noexception tensorflow::swig::RegisterSequenceClass; + +%unignore tensorflow::swig::IsSequence; +%noexception tensorflow::swig::IsSequence; + +%unignore tensorflow::swig::Flatten; +%noexception tensorflow::swig::Flatten; + +%include "tensorflow/python/util/util.h" + +%unignoreall diff --git a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc index bf81b9c0ad072823b0a64a2bc9e14e8feb96b0ba..00506fa54be93b54966a5b374b02286b7e784776 100644 --- a/tensorflow/stream_executor/cuda/cuda_diagnostics.cc +++ b/tensorflow/stream_executor/cuda/cuda_diagnostics.cc @@ -76,10 +76,10 @@ string DriverVersionStatusToString(port::StatusOr version) { port::StatusOr StringToDriverVersion(const string &value) { std::vector pieces = port::Split(value, '.'); - if (pieces.size() != 2 && pieces.size() != 3) { + if (pieces.size() < 2 || pieces.size() > 4) { return port::Status{ port::error::INVALID_ARGUMENT, - port::Printf("expected %%d.%%d or %%d.%%d.%%d form for driver version; got \"%s\"", + port::Printf("expected %%d.%%d, %%d.%%d.%%d, or %%d.%%d.%%d.%%d form for driver version; got \"%s\"", value.c_str())}; } diff --git a/tensorflow/stream_executor/cuda/cuda_platform.cc b/tensorflow/stream_executor/cuda/cuda_platform.cc index ee9df5b7de5c66d0f5ea41f4b6100dd58e89935c..874ac1ab6574bbf95b05893f34131b2cee9acc72 100644 --- a/tensorflow/stream_executor/cuda/cuda_platform.cc +++ b/tensorflow/stream_executor/cuda/cuda_platform.cc @@ -27,6 +27,42 @@ limitations under the License. namespace perftools { namespace gputools { namespace cuda { +namespace { + +// Synchronize with spinlocks. +const char kScheduleSpinString[] = "spin"; +// Synchronize with spinlocks that also call CPU yield instructions. +const char kScheduleYieldString[] = "yield"; +// Synchronize with a "synchronization primitive" (e.g. mutex). +const char kScheduleBlockingSyncString[] = "blocking_sync"; + +const DeviceOptions GetDeviceOptionsFromEnv() { + const char* gpu_schedule_string = + std::getenv("TF_CUDA_PLATFORM_GPU_DEVICE_SCHEDULE"); + + if (gpu_schedule_string == nullptr) { + return perftools::gputools::DeviceOptions::Default(); + } + + unsigned device_flags = 0; + if (strcmp(kScheduleSpinString, gpu_schedule_string) == 0) { + device_flags = perftools::gputools::DeviceOptions::kScheduleSpin; + } else if (strcmp(kScheduleYieldString, gpu_schedule_string) == 0) { + device_flags = perftools::gputools::DeviceOptions::kScheduleYield; + } else if (strcmp(kScheduleBlockingSyncString, gpu_schedule_string) == 0) { + device_flags = perftools::gputools::DeviceOptions::kScheduleBlockingSync; + } else { + LOG(QFATAL) << "Unknown option for environment variable " + "TF_CUDA_PLATFORM_GPU_DEVICE_SCHEDULE " + << gpu_schedule_string << " should be one of {" + << kScheduleBlockingSyncString << ", " << kScheduleSpinString + << ", " << kScheduleYieldString << "}"; + } + + return perftools::gputools::DeviceOptions(device_flags); +} + +} // namespace CudaPlatform::CudaPlatform() : name_("CUDA"), min_numa_node_(0), limit_numa_node_(0) {} @@ -112,7 +148,7 @@ port::StatusOr CudaPlatform::ExecutorForDevice(int ordinal) { StreamExecutorConfig config; config.ordinal = ordinal; config.plugin_config = PluginConfig(); - config.device_options = DeviceOptions::Default(); + config.device_options = GetDeviceOptionsFromEnv(); return GetExecutor(config); } @@ -121,7 +157,7 @@ port::StatusOr CudaPlatform::ExecutorForDeviceWithPluginConfig( StreamExecutorConfig config; config.ordinal = device_ordinal; config.plugin_config = plugin_config; - config.device_options = DeviceOptions::Default(); + config.device_options = GetDeviceOptionsFromEnv(); return GetExecutor(config); } diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 3001a37473715cb4dcf0cd133686eabfe171d402..e647a78055806186674eae2c3201e771ca9cbccb 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -1,13 +1,6 @@ # -*- Python -*- -# Given a source file, generate a test name. -# i.e. "common_runtime/direct_session_test.cc" becomes -# "common_runtime_direct_session_test" -def src_to_test_name(src): - return src.replace("/", "_").split(".")[0] - - # Return the options to use for a C++ library or binary build. # Uses the ":optmode" config_setting to pick the options. load( @@ -16,16 +9,30 @@ load( "tf_sycl_tests_tags", "tf_additional_xla_deps_py", "if_static",) -load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda", "cuda_default_copts") +load( + "@local_config_cuda//cuda:build_defs.bzl", + "if_cuda", + "cuda_default_copts",) load( "//third_party/mkl:build_defs.bzl", "if_mkl",) +def register_extension_info(**kwargs): + pass + + +# Given a source file, generate a test name. +# i.e. "common_runtime/direct_session_test.cc" becomes +# "common_runtime_direct_session_test" +def src_to_test_name(src): + return src.replace("/", "_").split(".")[0] + def full_path(relative_paths): return [PACKAGE_NAME + "/" + relative for relative in relative_paths] + # List of proto files for android builds def tf_android_core_proto_sources(core_proto_sources_relative): return [ @@ -290,6 +297,11 @@ def tf_cc_binary(name, linkopts=linkopts + _rpath_linkopts(name), **kwargs) +register_extension_info( + extension_name="tf_cc_binary", + label_regex_for_dep="{extension_name}.*") + + def tf_gen_op_wrapper_cc(name, out_ops_file, pkg="", @@ -551,6 +563,10 @@ def tf_cc_test(name, nocopts=nocopts, **kwargs) +register_extension_info( + extension_name="tf_cc_test", + label_regex_for_dep="{extension_name}.*") + # Part of the testing workflow requires a distinguishable name for the build # rules that involve a GPU, even if otherwise identical to the base rule. @@ -793,6 +809,11 @@ def tf_cuda_library(deps=None, cuda_deps=None, copts=None, **kwargs): copts=copts + if_cuda(["-DGOOGLE_CUDA=1"]) + if_mkl(["-DINTEL_MKL=1"]), **kwargs) +register_extension_info( + extension_name="tf_cuda_library", + label_regex_for_dep="{extension_name}") + + def tf_kernel_library(name, prefix=None, @@ -862,6 +883,10 @@ def tf_kernel_library(name, deps=deps, **kwargs) +register_extension_info( + extension_name="tf_kernel_library", + label_regex_for_dep="{extension_name}(_gpu)?") + def tf_mkl_kernel_library(name, prefix=None, @@ -1165,6 +1190,10 @@ def tf_custom_op_py_library(name, visibility=visibility, deps=deps,) +register_extension_info( + extension_name="tf_custom_op_py_library", + label_regex_for_dep="{extension_name}") + def tf_extension_linkopts(): return [] # No extension link opts @@ -1250,6 +1279,10 @@ def py_test(deps=[], **kwargs): }), **kwargs) +register_extension_info( + extension_name="py_test", + label_regex_for_dep="{extension_name}") + def tf_py_test(name, srcs, @@ -1284,6 +1317,10 @@ def tf_py_test(name, flaky=flaky, srcs_version="PY2AND3") +register_extension_info( + extension_name="tf_py_test", + label_regex_map={"deps": "additional_deps:{extension_name}"}) + def cuda_py_test(name, srcs, @@ -1310,6 +1347,10 @@ def cuda_py_test(name, flaky=flaky, xla_enabled=xla_enabled) +register_extension_info( + extension_name="cuda_py_test", + label_regex_map={"additional_deps": "additional_deps:{extension_name}"}) + def sycl_py_test(name, srcs, @@ -1336,6 +1377,10 @@ def sycl_py_test(name, flaky=flaky, xla_enabled=xla_enabled) +register_extension_info( + extension_name="sycl_py_test", + label_regex_map={"additional_deps": "additional_deps:{extension_name}"}) + def py_tests(name, srcs, diff --git a/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt b/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt index 5dd1ee47c969e1c31a0b44eb579ba255d49ebb46..6cac5c4d99fd7537b8fa852013ab348344be3f7e 100644 --- a/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.-optimizer-options.pbtxt @@ -46,6 +46,10 @@ tf_class { name: "Level" mtype: "" } + member { + name: "MAX_FOLDED_CONSTANT_IN_BYTES_FIELD_NUMBER" + mtype: "" + } member { name: "OFF" mtype: "" diff --git a/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt b/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt index 1e4d333cc0bb0bb33fb4cc8d76badd30c8babaa4..01cbd55c5d2e1b6fa3148af956217c3664864eaa 100644 --- a/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.bitwise.pbtxt @@ -16,4 +16,12 @@ tf_module { name: "invert" argspec: "args=[\'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "left_shift" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "right_shift" + argspec: "args=[\'x\', \'y\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } } diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt index 4c2dbc4d374cd628d932d7d6a4661e93fd4f25f6..ee37b1fa210ea816ef762590cfd1725c71262ed8 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-final-exporter.pbtxt @@ -9,7 +9,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " + argspec: "args=[\'self\', \'name\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\'], varargs=None, keywords=None, defaults=[\'None\', \'False\'], " } member_method { name: "export" diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt index ae1483bf3f055cd83151a2a53a404858abbc5700..2a9d0290295114daa006d39f17a295a01e40da6b 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-latest-exporter.pbtxt @@ -9,7 +9,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\', \'serving_input_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], " + argspec: "args=[\'self\', \'name\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'exports_to_keep\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'5\'], " } member_method { name: "export" diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt index 7ab094c99914d150323f3a581213dc1d82e8d4ab..d006ecb254724405bfec4000f063a93c41e77055 100644 --- a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt @@ -76,7 +76,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'100\', \'\', \'\', \'None\', \'5\', \'10000\', \'100\'], " + argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'\', \'\', \'None\', \'5\', \'10000\', \'100\'], " } member_method { name: "replace" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt index 43692a6c73e93366136cca82d828f8942bd0ebf1..a2b98b1c27c2268326af2653177b38e25f838c8d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.backend.name_scope.pbtxt @@ -2,6 +2,10 @@ path: "tensorflow.keras.backend.name_scope" tf_class { is_instance: "" is_instance: "" + member { + name: "name" + mtype: "" + } member_method { name: "__init__" argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt index c3d8893317a6839f9f848edd5c68555242b0b3c2..38e6128644529f012bdf1c9a7aa6656c1cef1ecd 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activation.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt index ea595964312edfd9ede4319b658ee3d6cb2d7c80..0fa60646612ab383a5022990c06b76571e269f05 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-activity-regularization.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt index 7e9b6bd70aa4b7356eb784f8b060d42d8fbc586c..75d56bf445847abfdc2b3e78d0ce5543aef152d9 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-add.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt index 804fb457841e4bcf8235f4595696b10d19aeb7b4..6e52b6238d5b255f75d1105f2e895267117a2029 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-alpha-dropout.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt index 6577856383b9bf2b52986a5adfe95c414cea12df..0e16774e8614e9b7ec7d4e90e176ba25f1512257 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt index fc4452948ada669f7d96a5da2b8d8767432d008a..98112762cf842519956af94ac8593c418e26c0d1 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt index ce19cea7cac08e197b24bfe685cb4391a12ca72c..2e093c0359664e8553c1be2c3b2d930df2c3aebe 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average-pooling3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt index 2ea54c2e3163adc717fac27087230c73f1735ec8..bada65e2f93cfa223b51d9ed3d44ab88cbad5a77 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-average.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt index 6fa1e153e034ac7408a64e992cf6dd6b44433592..120807c4b530c3fec508373bfc15131ffb532f72 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt index c6ff50bffcdca229555d36b20f3e6c62b076b077..834365f0f70e8447a8b6ba62cffe95a3c2a17e51 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt index 6d90a59d1e27c1ef8291f8426e804762d10bd67d..462a52ec1ebd4ce7f4b5289b76242ae1f992c032 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-avg-pool3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt index 278e5b583d935ebd4ae189077446a07c33fdec29..b802b363d013f819824c849ded762ff08a32cede 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-batch-normalization.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt index c9991db5c9c431295020390727bd52c9b35e7915..5279b2ab17d1fd3e8ca8cc75e9f7866ddaf25fb5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-bidirectional.pbtxt @@ -107,7 +107,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt index ec3c43945f7f0dbea66116d925301ed10f6d02d9..b800eb9796b04f0ffdb24768130669cec8e5babe 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-concatenate.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt index 2d6560828ecd2235fbef20b5192092d2d13d4731..a0906e62cf537b5d1b3c2c86e9b74f85df84022a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv-l-s-t-m2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt index f6f77ff80532f8b077a2d029d321eae470349634..47c63c11573e9fe20106f0a6a84a8940ae5f01e5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt index 854a06bf56a0e35de4fc3f95e4afe3fd85caa53e..e90b90e8016d4a955c57010cbf387d359963dafa 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d-transpose.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt index 5e71a9d355a18368ed9511ea7ea77287f93547a3..aa571b722dea6511925c9bf7f10714f252b897e7 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt index e7c98913fb31938383bdfac06330cd379f4a5973..911c73f8462df78c1353c5803660bffca4e33694 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d-transpose.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt index 3c4d078d1eb321092199ce34cc769ee84a54ec1f..bb111b327c22d6fdb502c8454fe114ee427d2a77 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-conv3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt index 8043eb0610192240ee2eb6dfb4912993507ba55f..5a5ec635cce36c4e4561f73700e73a3ae215c596 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt index a9a90891a438cfdc78b2c2813e39a06a3122a599..190b670fa2a4b04e124c3d1f63e691dfbe8cdbbb 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d-transpose.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt index dae5a66190089bd6d2ab3e60cb777db8a1790b41..a26ec82f2b96e69c445f05aa852a7b37ab67dbd7 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt index 37aa80eb704c157528e4bb187f4f5f90c9ef08ce..19b5bdf36befbcf8877fc28b54d9c712d83a74b2 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d-transpose.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt index fa28ce17ecca6e388c4dcb84830a4a39e30153a5..773ef01feb8fe179ec34d3e392395afb79200b8f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-convolution3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt index 8e2b530d0812448fac3111f47cf417e83edec547..3a67ac00ab193b3e7e72a105b6df2757c0164b74 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt index 70b1c50a0ab5645ad328b3fcff7d1fb1039c927d..de5a695b69f5b6977546fbf6211b26973c47fda2 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt index 1b2b4e934dfd4d71cb329033c1ce64072c7e8820..bf251b4df5dae7b9541679062ef5fa163e22bda5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-cropping3-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt index fb0fcd26143814ef87cba6dec745ce5446f929e7..92a74cec68090271a59dd44dd93c1fc4821afbdd 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dense.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt index af8ad3abaadb176185d624322905c2bd6e081d5b..cdd62eee0d3a26303caaa0f643b9fedae81f91d7 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dot.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt index e774a4d412b6a99f0f4e3355001e2b575c0f1f50..7935143b2cf4f84dfd7d81286dea96ca9f57ec6c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-dropout.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt index 46eb76720869c398e15e77a4e21d49fb261afb4a..497eb004992e4256fec158e1eb50dbb0b915aeea 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-e-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt index 5e74cb6970b07f8daa99a4cbad7fc6e91e61d576..35616cbebb388f1198aebdf0eeb5eaed76ea52e1 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-embedding.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt index a4c8759a2cbed61b5570d805bec394ef672caef8..427c6fde90334a39cd1e3bef96952c792a1d3955 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-flatten.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt index 9738dd004ac93f73fea878e6208b4f48b35b05ca..92373992548e3ea48ae54d1cad0a81ebd4966b1d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-g-r-u.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt index ce033eaa00d3f6ea97eac48911b2791ddbd0f2f3..1428691afe2a525cc46ddb4f1b73239cdb613b31 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-dropout.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt index 4cd6d714a0626bc198104d10ed8ad3d214057288..655734cc432f7f18dfc5dc1f5f255650cb574a1a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-gaussian-noise.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt index 2bd80f97ae88cd47d0d38de906e5e6ca3c2dd3ad..d97f06ea137f1128ffb6e1ebfcc10e0160904387 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt index a9d00fd7c126378ac3962a4f334605d7d720f9e7..52886b2106aa3f99ded8a66a20f5cf6bec48b233 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt index a2b00778fe12698523e6064f0f565cbb9eca6d1c..ccb6459357f8248cc760995de94f5ef305d8c64b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-average-pooling3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt index 01a9839cccea3affd32f3c033b85376ac26acfd2..1f25eb1cc64fcfe3489fef1c32f1b806ca74b478 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt index b041dfc71ef5d288b6b2c992827de2a2dd32ac25..a37d6dda28a653836bf4c495165f2aff05744298 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt index 6ba06a4e7e9fd142963fdeffac0be6f16b4a7997..9f276fd54714756d6db17921c7f4f139a8b05a8b 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-avg-pool3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt index fb62a3e035b4be9453f442b29a4f06b80b4f3883..eaa9b477d853c12e7b7dd183e09073a8116b24e6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt index 3d1c66441cf7bce2203f6816f0224246d4a04723..f4d37a5f63432dec1131bc7bce0cabd1af6e8db3 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt index d55a82e0a34251104ebed4ca44087dad5f32d685..afddd2d4cbf7d1ba089fbc35a684664aded4e2a6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pool3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt index 70177c8623886441e681c8c86f1e594cec8abd16..12cd49c9551c4520fd05592d7a3f456b3d328859 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling1-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt index da231a4fceec5dfd82489282db901b163105580a..146241c172319211827d77482c634ae6218137b0 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling2-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt index aa3eb1c70480eb9d9e2036655a9f6e73f9e8c064..00475301aa009385e7f23241864475f38bf00da8 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-global-max-pooling3-d.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt index 40f0f7c8000a422ca8edfeadb0a09837ad26e146..b2df5fba8fd748f43a3b88aee0993e1f5262d724 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-input-layer.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt index 1a9ec4a506c7f00230df2387cb94d64dcd4e02ad..20935e2f99a8a7a5054cda50e3b38442a216377f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-l-s-t-m.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt index 69086963b656bb87cd6ffb12047fd03c812f7b47..59508c2f11073caca1f30544efaea435730ce228 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-lambda.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt index d350a52171d58bdfdbc5c8f4abf9d32f50e74c4d..ca904a2b8c77e55430e4f76ff4fa2be641c199a6 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-layer.pbtxt @@ -101,7 +101,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt index 05952c1d96e15a5a5912c68c461a8eb39c676a63..f52fd02515f30f4011a154cda4274d7e7dd34a88 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-leaky-re-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt index c49b8de5fb0920e5aca72ee4f47b0092193726fb..b5c32d1cdf3dc7e35d9c78dd81431bb67aab1b27 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt index e24e3697b2cb65a14bf9bd2caacc672f05b560bd..0ac2b83a999b3c8245ce616ccf5d79833747aee3 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-locally-connected2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt index 246340a1cee3afcc2b142ef376104af499002453..de2a28d985d3f05c639a103b316f66f15d326f95 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-masking.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt index eb631b1d3852e856224e3ac954382a074d6c2994..130d932fd6de0fde1052843cd9b10bf2a748441c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt index cfe6af339e7401f5746e3ca8979f88f8bf1ca437..82a6f6d539080436ef2e49b5a4b342a2dfff3ae2 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt index 4bb5a23927015330404bcb50aba28cc88da9b7f2..ca2fd4e502b6bb87c44363f91f9dcd26b386eb3a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pool3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt index 6c9b9a92eb5059495e1919b062d1837eeb91cedb..885e30f8799fd7e156c9f048b59483ad00d41fba 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt index cdc4c43ad6df11477b8071407f748702116a9983..102879d2f536cc6bfdb31558c36412b3d1e93885 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt index 4959dc58d14aa18a703c48c93e9c61fc18b36eba..424061614659249605569a571ff09adf52db3997 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-max-pooling3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt index 7ff5ee02e10534f4455f570e7258f3e335f13f06..4b32c2e99f9a0e3abdb3f1bfe27137f6f5052491 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-maximum.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt index 860ebd509b9da9e50307e8a6df4329ab4225dd57..0c964235ae7d2c352c053b97f902bf2516263628 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-multiply.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt index e32800bd252f87f3080ed8e9790b072a01d4abd7..797a073b8a74bc482a555bb12806afe36d0df79e 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-p-re-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt index 8b453f7a1bd358612669b56bbbc675ff4539fc11..7dc1fa6964eb0fcba47ae2db270152364e244eaa 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-permute.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt index 9b53609e4d4040267452dec67a3d25ce3dc09a86..dedb48151a84f00f96db8942e08f5508cecfcbba 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-repeat-vector.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt index f7a774a38f88b3582688f9928be61029e6128e2c..bb30c0a945da7b6f869fa385eebbb8301851e8ae 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-reshape.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt index 4f1d2db4ccd40805dba07d28478040d1193516b9..7867e3c1fd3c670f3973a15047e04fc2aece0f86 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-conv2-d.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt index 066519cba858267fa94ea58b51a9fffe97cd81ab..0fb6e84f8deeb9459d5cce6a4565da61304b6ca5 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-separable-convolution2-d.pbtxt @@ -105,7 +105,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt index 6a08eb785b3b0708141171a8c994f288d654a1d7..f4148fcc2309f77c804fc853b1a0d8fda02d063a 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-simple-r-n-n.pbtxt @@ -103,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt index b85003d52edd24bc92efa7050ea775b3842f33e3..9773c4acc750c59a810cc467a9239a397c62ec25 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout1-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt index 83d4258a66294ad69260d3be6d2f127cfc831ec8..d4de587a4801c56ca5903bdd1169b816d008765d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout2-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt index a49060b86023017db21256ea7c9ee89fcbdf833c..af210fab8dc444bfb3b3f8fda0edb5121f6ad0ba 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-spatial-dropout3-d.pbtxt @@ -104,7 +104,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt index 01b91b9bbcef113f5a415bb8a05179e3d51d913f..8cfb33a14896b767deae34d4b76485729ef0122f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-thresholded-re-l-u.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt index 4713bd16e1f5ab8a4204d5bb779bbbaae6e954b3..34c9efb3ca00a3b37fa6f05a4ea58cff89ccbcdf 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-time-distributed.pbtxt @@ -107,7 +107,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt index 393980ecde8ea08836f2165ea4638adf0bc09536..bb42cdcb65643190f1d634e2ad23447fb40c90ee 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt index 7ddb282f069ffa03bac37360f6dd4af41263f1f9..6d3c2ebfef8af42c288d7de6124e1ae326994c1d 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt index c1bd2dcbaf35751e4355de2a6ee9d15dabbf7484..d790cf2e08030d3b3f362a19474fd6d1d7833c65 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-up-sampling3-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt index c020dc3954e87bb833229bde816182ec9cf9af12..9cee68874a9e32a9aa4c0086a6b473c347446f8c 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-wrapper.pbtxt @@ -106,7 +106,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt index b7fe482145b3125c8da621bed18f0987abed12a1..ba6c23ae75afae9177fa4f1fda34dc3f6d12939e 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding1-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt index 51f50882b2025a6c0fa28a87e3312e6a46317ef9..cb587d67b0d99cf38823c2d74b833474ec4b5b10 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding2-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt index e558931ead6a2b249f77a7d5f102ce134c2729c5..415720cbe11134f6b2426a2eab395566e65cbf8f 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.layers.-zero-padding3-d.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt index 1f3422b9a1c52420bcef42099a21acb86bcdea11..af9a44086fd618e559d807a98e145c6f1d423156 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-model.pbtxt @@ -120,7 +120,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt index 187c3a85b3dbf4c5d6482bc7b57f9e4bee1e00b1..5034fdff2a6bd78e9bad0403d4c33d72c1b766af 100644 --- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt @@ -133,7 +133,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "add_weight" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt index 7fdf97ed79a6b2fe9a0eaee1cc77195d08f854ab..6e595ca34385d14f3ea7eb0da9a633f6f308f72f 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling1-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt index 5911fbefa9d4360316c632e7fc91416e937b7b46..7b6c30773b95984ee8438820a45bf2c607a912ff 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling2-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt index e8374586154b04d71212576b1234130e995418e9..7a7664e80013557c922a1d399de16e32a78f60ff 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-average-pooling3-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt index c66af13850b53789099c142412e369da5d066914..c9f5c18f25628d8b1d575113232da8a75d0e428c 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-batch-normalization.pbtxt @@ -81,7 +81,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'trainable\', \'virtual_batch_size\', \'adjustment\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt index 9ee79be96d0df0c9b6480c58531bc10735dabf95..1fa00d7b2f9d3d34861e2030a98487d660e81305 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv1-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt index 67bd7d2cc149598633a18800d89ffb73af5969b5..a92a1094ac0c042e6ab9a2d153e8a06ab183d0ec 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d-transpose.pbtxt @@ -95,7 +95,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt index f310b7ea86b46d7d97c96fe7a0a1e22c4a69c373..7fa78ab20b1260c1eb87293e200015c7b2895b19 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv2-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt index b78666779525c4663a0a9c48fd252eb9ad535057..e92e4859ae5b179f8b2a2328219aa6f16d740903 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d-transpose.pbtxt @@ -95,7 +95,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt index 02c8130b485fd53ec16f296a57eff68c9fe3c6c3..87e5c2949e681e224efc94265559c31256082f1e 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-conv3-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt index 268cb788d14ab1ca0ebe82eb86ced481221327df..cc4ee4c8a5ec6bc8f2395fedb8aed8d334342013 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-dense.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt index 969ec33578884a1f1364fe0489c0e9ed31b1de9f..99ab2ef97c73bbd305a3755b78e8174b643fe0a0 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-dropout.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt index fb602e41be8d7ea46584400736ac5b055b382d81..f4074c5a4f6b45896f49e295830c91f58b46c84a 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-flatten.pbtxt @@ -93,7 +93,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt index ec65fc4555fa70bd5e47d422228136561478e5ed..ec51609dee9bedb75566163e35225a1797d4cd5c 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-layer.pbtxt @@ -92,7 +92,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt index 60aec6cd149db33e478ed1c3ca556cf6b5ebe68b..745c532e94beda1280e366bf592d347a5275ad11 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling1-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt index bc2f49cc181f30cdbc7003c33f9c7e866bde8633..f8244c01b64105d0c4467c3f90ccec4e2d06adb4 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling2-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt index 83b98059f9ec3dd275240ac212f040d3c39f7644..df5378f279bbeb254f4a9fee2724b07baee87203 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-max-pooling3-d.pbtxt @@ -94,7 +94,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt index 83f3ed82daef1448efa6d760947615c66b6c83fc..c55d2bccc9b10d142d82073013639121dc45ebf1 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.-separable-conv2-d.pbtxt @@ -95,7 +95,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt index dad514b534ae39492cb3d1cb9a6d8f0079653abf..c45d6e6c05054f1c0c61caeaf5e9a3fd7d00983f 100644 --- a/tensorflow/tools/api/golden/tensorflow.layers.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.layers.pbtxt @@ -90,7 +90,7 @@ tf_module { } member_method { name: "batch_normalization" - argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\'], " + argspec: "args=[\'inputs\', \'axis\', \'momentum\', \'epsilon\', \'center\', \'scale\', \'beta_initializer\', \'gamma_initializer\', \'moving_mean_initializer\', \'moving_variance_initializer\', \'beta_regularizer\', \'gamma_regularizer\', \'beta_constraint\', \'gamma_constraint\', \'training\', \'trainable\', \'name\', \'reuse\', \'renorm\', \'renorm_clipping\', \'renorm_momentum\', \'fused\', \'virtual_batch_size\', \'adjustment\'], varargs=None, keywords=None, defaults=[\'-1\', \'0.99\', \'0.001\', \'True\', \'True\', \'\', \'\', \'\', \'\', \'None\', \'None\', \'None\', \'None\', \'False\', \'True\', \'None\', \'None\', \'False\', \'None\', \'0.99\', \'None\', \'None\', \'None\'], " } member_method { name: "conv1d" diff --git a/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt b/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt index daa3785034d342280f62d0891b7314564785b347..2aab2c4a778049d9ac7bfd2adb5950afa50396f1 100644 --- a/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.metrics.pbtxt @@ -68,6 +68,10 @@ tf_module { name: "precision_at_thresholds" argspec: "args=[\'labels\', \'predictions\', \'thresholds\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " } + member_method { + name: "precision_at_top_k" + argspec: "args=[\'labels\', \'predictions_idx\', \'k\', \'class_id\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "recall" argspec: "args=[\'labels\', \'predictions\', \'weights\', \'metrics_collections\', \'updates_collections\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt index 107f066c29c13723e717fea78685d1d70347df13..80418970132377a5d578e4f11fa4091a19202cf3 100644 --- a/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.name_scope.pbtxt @@ -2,6 +2,10 @@ path: "tensorflow.name_scope" tf_class { is_instance: "" is_instance: "" + member { + name: "name" + mtype: "" + } member_method { name: "__init__" argspec: "args=[\'self\', \'name\', \'default_name\', \'values\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt index f10299377b32bc7fcd3c9e489199f13076a52690..11637814a6e5591668d9f3594898bd6123b9edd6 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.pbtxt @@ -4,6 +4,10 @@ tf_module { name: "rnn_cell" mtype: "" } + member { + name: "swish" + mtype: "" + } member_method { name: "all_candidate_sampler" argspec: "args=[\'true_classes\', \'num_true\', \'num_sampled\', \'unique\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt index 3254a62af156a119a5f134dc2a3bd1ad647ea6d0..49066eecaa0fda4a7a60c62b7a087d054bd73079 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-l-s-t-m-cell.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.nn.rnn_cell.BasicLSTMCell" tf_class { is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -90,7 +91,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'num_units\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\'], varargs=None, keywords=None, defaults=[\'1.0\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'num_units\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -102,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,7 +111,7 @@ tf_class { } member_method { name: "build" - argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None" } member_method { name: "call" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt index 29bc20ef1ae80fd32b603731f4c47f60eabf2031..5646461b24de2cd73eacf89cdf7611d34af70445 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-basic-r-n-n-cell.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt index 17ee1ff5fbf6a680c12b29502b87695e48181dba..81dcd90e81e9185f087892a5ebda0bb8460b0d8d 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-device-wrapper.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt index fe4f630a39acd2d625c3dfe8aa03c79f966ab943..8ff225897ae26adb3723aaf729030771e26833a4 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-dropout-wrapper.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt index 1c8dd65d27f06c80975cac0b4e9c45eccda4c233..2adfc747d1939d38f526487082e9d3e5e9b24eae 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-g-r-u-cell.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt index 0f294e216a57de9767458f8e238fff9f36fce385..8d17153972cfd99072eee1db56728e67b98db0da 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-l-s-t-m-cell.pbtxt @@ -1,6 +1,7 @@ path: "tensorflow.nn.rnn_cell.LSTMCell" tf_class { is_instance: "" + is_instance: "" is_instance: "" is_instance: "" is_instance: "" @@ -90,7 +91,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'num_units\', \'use_peepholes\', \'cell_clip\', \'initializer\', \'num_proj\', \'proj_clip\', \'num_unit_shards\', \'num_proj_shards\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'1.0\', \'True\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'num_units\', \'use_peepholes\', \'cell_clip\', \'initializer\', \'num_proj\', \'proj_clip\', \'num_unit_shards\', \'num_proj_shards\', \'forget_bias\', \'state_is_tuple\', \'activation\', \'reuse\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'1.0\', \'True\', \'None\', \'None\', \'None\'], " } member_method { name: "add_loss" @@ -102,7 +103,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" @@ -110,7 +111,7 @@ tf_class { } member_method { name: "build" - argspec: "args=[\'self\', \'_\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'inputs_shape\'], varargs=None, keywords=None, defaults=None" } member_method { name: "call" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt index ed42631471e6ed1af32f2e35e423d098d836813a..68c3064dd4f2f1453102cffd078e6a2e5356e0d5 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-multi-r-n-n-cell.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt index 2c7dc7c4f2d79851699da00cc80af360cdb087da..86ff0fee2b369fb77bdcba6b19dc89f39a48642b 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-r-n-n-cell.pbtxt @@ -101,7 +101,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt index dbcbf29586e0694320ac4bb23e9b85f95f39790c..1a6f8a3b7dc1990b83f518ee1970ab36b2594fda 100644 --- a/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.nn.rnn_cell.-residual-wrapper.pbtxt @@ -102,7 +102,7 @@ tf_class { } member_method { name: "add_variable" - argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\'], " + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'partitioner\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\', \'None\', \'None\'], " } member_method { name: "apply" diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt index d56a59de7230eeb4ceba72f130837aca9440f6ef..bf7bc6a7c1556db1097e518c4d2d3ce26a4ce208 100644 --- a/tensorflow/tools/api/golden/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.pbtxt @@ -16,6 +16,10 @@ tf_module { name: "COMPILER_VERSION" mtype: "" } + member { + name: "CXX11_ABI_FLAG" + mtype: "" + } member { name: "ConditionalAccumulator" mtype: "" @@ -1480,9 +1484,13 @@ tf_module { name: "qr" argspec: "args=[\'input\', \'full_matrices\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "quantize" + argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'round_mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'HALF_AWAY_FROM_ZERO\', \'None\'], " + } member_method { name: "quantize_v2" - argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\'], " + argspec: "args=[\'input\', \'min_range\', \'max_range\', \'T\', \'mode\', \'name\', \'round_mode\'], varargs=None, keywords=None, defaults=[\'MIN_COMBINED\', \'None\', \'HALF_AWAY_FROM_ZERO\'], " } member_method { name: "quantized_concat" diff --git a/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt b/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt index 02dec04b9ccdb4ddf38ffee6e3a81617245b123d..2f00aeac25f691d9767080251798248281e5edf5 100644 --- a/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.sysconfig.pbtxt @@ -1,5 +1,9 @@ path: "tensorflow.sysconfig" tf_module { + member_method { + name: "get_compile_flags" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } member_method { name: "get_include" argspec: "args=[], varargs=None, keywords=None, defaults=None" @@ -8,4 +12,8 @@ tf_module { name: "get_lib" argspec: "args=[], varargs=None, keywords=None, defaults=None" } + member_method { + name: "get_link_flags" + argspec: "args=[], varargs=None, keywords=None, defaults=None" + } } diff --git a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..03efe6639e0e3d2c6c280bd30d2b59b5d654f995 --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.-step-context.pbtxt @@ -0,0 +1,21 @@ +path: "tensorflow.train.MonitoredSession.StepContext" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "session" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "request_stop" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "run_with_hooks" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt index 3a5cc015b4d5a0ca3487764787bc877716d9fedc..09b7b3fb538fb8d87dcfd622089818081a1fb79b 100644 --- a/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.train.-monitored-session.pbtxt @@ -3,6 +3,10 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "StepContext" + mtype: "" + } member { name: "graph" mtype: "" @@ -19,6 +23,10 @@ tf_class { name: "run" argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } + member_method { + name: "run_step_fn" + argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "should_stop" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt new file mode 100644 index 0000000000000000000000000000000000000000..36d8ce7ff82e02300b59705400be40d7cc3f65ae --- /dev/null +++ b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.-step-context.pbtxt @@ -0,0 +1,21 @@ +path: "tensorflow.train.SingularMonitoredSession.StepContext" +tf_class { + is_instance: "" + is_instance: "" + member { + name: "session" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'session\', \'run_with_hooks_fn\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "request_stop" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "run_with_hooks" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt index 7caf837cc385dbd64611a58de2c25d4de221a911..de0f2c1c1a2497ef4e541ee6583d416e31f48826 100644 --- a/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.train.-singular-monitored-session.pbtxt @@ -3,6 +3,10 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "StepContext" + mtype: "" + } member { name: "graph" mtype: "" @@ -23,6 +27,10 @@ tf_class { name: "run" argspec: "args=[\'self\', \'fetches\', \'feed_dict\', \'options\', \'run_metadata\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], " } + member_method { + name: "run_step_fn" + argspec: "args=[\'self\', \'step_fn\'], varargs=None, keywords=None, defaults=None" + } member_method { name: "should_stop" argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" diff --git a/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt index cc9bd5c136bcedd6345a64db165ff6e847b20d3a..1f0e59a1ac2d899a50ff30c7c8da8f91a0258a1e 100644 --- a/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.train.-supervisor.pbtxt @@ -88,7 +88,7 @@ tf_class { } member_method { name: "Stop" - argspec: "args=[\'self\', \'threads\', \'close_summary_writer\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], " + argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], " } member_method { name: "StopOnException" @@ -136,7 +136,7 @@ tf_class { } member_method { name: "stop" - argspec: "args=[\'self\', \'threads\', \'close_summary_writer\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], " + argspec: "args=[\'self\', \'threads\', \'close_summary_writer\', \'ignore_live_threads\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'False\'], " } member_method { name: "stop_on_exception" diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt index edc29e62dd02ba070e5249132b95f40c8a1ff2b3..e73f6f6e6323c45d0f581efc4c5ae3615859d182 100644 --- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt +++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt @@ -264,6 +264,10 @@ tf_module { name: "checkpoint_exists" argspec: "args=[\'checkpoint_prefix\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "cosine_decay" + argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "create_global_step" argspec: "args=[\'graph\'], varargs=None, keywords=None, defaults=[\'None\'], " @@ -328,6 +332,10 @@ tf_module { name: "limit_epochs" argspec: "args=[\'tensor\', \'num_epochs\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], " } + member_method { + name: "linear_cosine_decay" + argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'0.5\', \'0.0\', \'0.001\', \'None\'], " + } member_method { name: "list_variables" argspec: "args=[\'ckpt_dir_or_file\'], varargs=None, keywords=None, defaults=None" @@ -364,6 +372,10 @@ tf_module { name: "natural_exp_decay" argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'decay_rate\', \'staircase\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], " } + member_method { + name: "noisy_linear_cosine_decay" + argspec: "args=[\'learning_rate\', \'global_step\', \'decay_steps\', \'initial_variance\', \'variance_decay\', \'num_periods\', \'alpha\', \'beta\', \'name\'], varargs=None, keywords=None, defaults=[\'1.0\', \'0.55\', \'0.5\', \'0.0\', \'0.001\', \'None\'], " + } member_method { name: "piecewise_constant" argspec: "args=[\'x\', \'boundaries\', \'values\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD index e99cc0572f8206080db75d168e6af0ab4b691901..f80dd6fe5b6a70b4198fff8da7b457645452b3e1 100644 --- a/tensorflow/tools/api/tests/BUILD +++ b/tensorflow/tools/api/tests/BUILD @@ -11,10 +11,16 @@ exports_files([ "API_UPDATE_WARNING.txt", ]) +load("//tensorflow:tensorflow.bzl", "tf_cc_binary") + py_test( name = "api_compatibility_test", srcs = ["api_compatibility_test.py"], data = [ + ":convert_from_multiline", + "//tensorflow/core:base_api_def", + "//tensorflow/core:python_api_def", + "//tensorflow/python:hidden_ops", "//tensorflow/tools/api/golden:api_golden", "//tensorflow/tools/api/tests:API_UPDATE_WARNING.txt", "//tensorflow/tools/api/tests:README.txt", @@ -23,6 +29,7 @@ py_test( deps = [ "//tensorflow:tensorflow_py", "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_test_lib", "//tensorflow/python:lib", "//tensorflow/python:platform", "//tensorflow/tools/api/lib:python_object_to_proto_visitor", @@ -31,6 +38,15 @@ py_test( ], ) +tf_cc_binary( + name = "convert_from_multiline", + srcs = ["convert_from_multiline.cc"], + deps = [ + "//tensorflow/core:lib", + "//tensorflow/core:op_gen_lib", + ], +) + filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py index 1ffa8fc26c0c69e372f8610feb736a8365835d57..6a27f6bc42fb3205b95384b66cb9d0f29f26fa55 100644 --- a/tensorflow/tools/api/tests/api_compatibility_test.py +++ b/tensorflow/tools/api/tests/api_compatibility_test.py @@ -28,8 +28,11 @@ from __future__ import division from __future__ import print_function import argparse +from collections import defaultdict +from operator import attrgetter import os import re +import subprocess import sys import unittest @@ -37,6 +40,7 @@ import tensorflow as tf from google.protobuf import text_format +from tensorflow.core.framework import api_def_pb2 from tensorflow.python.lib.io import file_io from tensorflow.python.platform import resource_loader from tensorflow.python.platform import test @@ -64,6 +68,12 @@ _API_GOLDEN_FOLDER = 'tensorflow/tools/api/golden' _TEST_README_FILE = 'tensorflow/tools/api/tests/README.txt' _UPDATE_WARNING_FILE = 'tensorflow/tools/api/tests/API_UPDATE_WARNING.txt' +_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' +_CONVERT_FROM_MULTILINE_SCRIPT = 'tensorflow/tools/api/tests/convert_from_multiline' +_BASE_API_DIR = 'tensorflow/core/api_def/base_api' +_PYTHON_API_DIR = 'tensorflow/core/api_def/python_api' +_HIDDEN_OPS_FILE = 'tensorflow/python/ops/hidden_ops.txt' + def _KeyToFilePath(key): """From a given key, construct a filepath.""" @@ -88,6 +98,45 @@ def _FileNameToKey(filename): return api_object_key +def _GetSymbol(symbol_id): + """Get TensorFlow symbol based on the given identifier. + + Args: + symbol_id: Symbol identifier in the form module1.module2. ... .sym. + + Returns: + Symbol corresponding to the given id. + """ + # Ignore first module which should be tensorflow + symbol_id_split = symbol_id.split('.')[1:] + symbol = tf + for sym in symbol_id_split: + symbol = getattr(symbol, sym) + return symbol + + +def _IsGenModule(module_name): + if not module_name: + return False + module_name_split = module_name.split('.') + return module_name_split[-1].startswith('gen_') + + +def _GetHiddenOps(): + hidden_ops_file = file_io.FileIO(_HIDDEN_OPS_FILE, 'r') + hidden_ops = set() + for line in hidden_ops_file: + line = line.strip() + if not line: + continue + if line[0] == '#': # comment line + continue + # If line is of the form "op_name # comment", only keep the op_name. + line_split = line.split('#') + hidden_ops.add(line_split[0].strip()) + return hidden_ops + + class ApiCompatibilityTest(test.TestCase): def __init__(self, *args, **kwargs): @@ -229,6 +278,176 @@ class ApiCompatibilityTest(test.TestCase): update_goldens=FLAGS.update_goldens) +class ApiDefTest(test.TestCase): + + def __init__(self, *args, **kwargs): + super(ApiDefTest, self).__init__(*args, **kwargs) + self._first_cap_pattern = re.compile('(.)([A-Z][a-z]+)') + self._all_cap_pattern = re.compile('([a-z0-9])([A-Z])') + + def _GenerateLowerCaseOpName(self, op_name): + lower_case_name = self._first_cap_pattern.sub(r'\1_\2', op_name) + return self._all_cap_pattern.sub(r'\1_\2', lower_case_name).lower() + + def _CreatePythonApiDef(self, base_api_def, endpoint_names): + """Creates Python ApiDef that overrides base_api_def if needed. + + Args: + base_api_def: (api_def_pb2.ApiDef) base ApiDef instance. + endpoint_names: List of Python endpoint names. + + Returns: + api_def_pb2.ApiDef instance with overrides for base_api_def + if module.name endpoint is different from any existing + endpoints in base_api_def. Otherwise, returns None. + """ + endpoint_names_set = set(endpoint_names) + base_endpoint_names_set = { + self._GenerateLowerCaseOpName(endpoint.name) + for endpoint in base_api_def.endpoint} + + if endpoint_names_set == base_endpoint_names_set: + return None # All endpoints are the same + + api_def = api_def_pb2.ApiDef() + api_def.graph_op_name = base_api_def.graph_op_name + + for endpoint_name in sorted(endpoint_names): + new_endpoint = api_def.endpoint.add() + new_endpoint.name = endpoint_name + + return api_def + + def _GetBaseApiMap(self): + """Get a map from graph op name to its base ApiDef. + + Returns: + Dictionary mapping graph op name to corresponding ApiDef. + """ + # Convert base ApiDef in Multiline format to Proto format. + converted_base_api_dir = os.path.join( + test.get_temp_dir(), 'temp_base_api_defs') + subprocess.check_call( + [os.path.join(resource_loader.get_root_dir_with_all_resources(), + _CONVERT_FROM_MULTILINE_SCRIPT), + _BASE_API_DIR, converted_base_api_dir]) + + name_to_base_api_def = {} + base_api_files = file_io.get_matching_files( + os.path.join(converted_base_api_dir, 'api_def_*.pbtxt')) + for base_api_file in base_api_files: + if file_io.file_exists(base_api_file): + api_defs = api_def_pb2.ApiDefs() + text_format.Merge( + file_io.read_file_to_string(base_api_file), api_defs) + for api_def in api_defs.op: + name_to_base_api_def[api_def.graph_op_name] = api_def + return name_to_base_api_def + + def _AddHiddenOpOverrides(self, name_to_base_api_def, api_def_map): + """Adds ApiDef overrides to api_def_map for hidden Python ops. + + Args: + name_to_base_api_def: Map from op name to base api_def_pb2.ApiDef. + api_def_map: Map from first op name character (in caps) to + api_def_pb2.ApiDefs for Python API overrides. + """ + hidden_ops = _GetHiddenOps() + for hidden_op in hidden_ops: + if hidden_op not in name_to_base_api_def: + logging.warning('Unexpected hidden op name: %s' % hidden_op) + continue + + base_api_def = name_to_base_api_def[hidden_op] + if base_api_def.visibility != api_def_pb2.ApiDef.HIDDEN: + api_def = api_def_pb2.ApiDef() + api_def.graph_op_name = base_api_def.graph_op_name + api_def.visibility = api_def_pb2.ApiDef.HIDDEN + api_def_map[api_def.graph_op_name[0].upper()].op.extend([api_def]) + + @unittest.skipUnless( + sys.version_info.major == 2 and os.uname()[0] == 'Linux', + 'API compabitility test goldens are generated using python2 on Linux.') + def testAPIDefCompatibility(self): + # Get base ApiDef + name_to_base_api_def = self._GetBaseApiMap() + snake_to_camel_graph_op_names = { + self._GenerateLowerCaseOpName(name): name + for name in name_to_base_api_def.keys()} + # Extract Python API + visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor() + public_api_visitor = public_api.PublicAPIVisitor(visitor) + public_api_visitor.do_not_descend_map['tf'].append('contrib') + traverse.traverse(tf, public_api_visitor) + proto_dict = visitor.GetProtos() + + # Map from first character of op name to Python ApiDefs. + api_def_map = defaultdict(api_def_pb2.ApiDefs) + # We need to override all endpoints even if 1 endpoint differs from base + # ApiDef. So, we first create a map from an op to all its endpoints. + op_to_endpoint_name = defaultdict(list) + + # Generate map from generated python op to endpoint names. + for public_module, value in proto_dict.items(): + module_obj = _GetSymbol(public_module) + for sym in value.tf_module.member_method: + obj = getattr(module_obj, sym.name) + + # Check if object is defined in gen_* module. That is, + # the object has been generated from OpDef. + if hasattr(obj, '__module__') and _IsGenModule(obj.__module__): + if obj.__name__ not in snake_to_camel_graph_op_names: + # Symbol might be defined only in Python and not generated from + # C++ api. + continue + relative_public_module = public_module[len('tensorflow.'):] + full_name = (relative_public_module + '.' + sym.name + if relative_public_module else sym.name) + op_to_endpoint_name[obj].append(full_name) + + # Generate Python ApiDef overrides. + for op, endpoint_names in op_to_endpoint_name.items(): + graph_op_name = snake_to_camel_graph_op_names[op.__name__] + api_def = self._CreatePythonApiDef( + name_to_base_api_def[graph_op_name], endpoint_names) + if api_def: + api_defs = api_def_map[graph_op_name[0].upper()] + api_defs.op.extend([api_def]) + + self._AddHiddenOpOverrides(name_to_base_api_def, api_def_map) + + for key in _ALPHABET: + # Get new ApiDef for the given key. + new_api_defs_str = '' + if key in api_def_map: + new_api_defs = api_def_map[key] + new_api_defs.op.sort(key=attrgetter('graph_op_name')) + new_api_defs_str = str(new_api_defs) + + # Get current ApiDef for the given key. + api_defs_file_path = os.path.join( + _PYTHON_API_DIR, 'api_def_%s.pbtxt' % key) + old_api_defs_str = '' + if file_io.file_exists(api_defs_file_path): + old_api_defs_str = file_io.read_file_to_string(api_defs_file_path) + + if old_api_defs_str == new_api_defs_str: + continue + + if FLAGS.update_goldens: + if not new_api_defs_str: + logging.info('Deleting %s...' % api_defs_file_path) + file_io.delete_file(api_defs_file_path) + else: + logging.info('Updating %s...' % api_defs_file_path) + file_io.write_string_to_file(api_defs_file_path, new_api_defs_str) + else: + self.assertMultiLineEqual( + old_api_defs_str, new_api_defs_str, + 'To update golden API files, run api_compatibility_test locally ' + 'with --update_goldens=True flag.') + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( diff --git a/tensorflow/tools/api/tests/convert_from_multiline.cc b/tensorflow/tools/api/tests/convert_from_multiline.cc new file mode 100644 index 0000000000000000000000000000000000000000..5c5aaa4f06fff2af4b9683255aa15aaf7d2172e3 --- /dev/null +++ b/tensorflow/tools/api/tests/convert_from_multiline.cc @@ -0,0 +1,63 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// Converts all *.pbtxt files in a directory from Multiline to proto format. +#include "tensorflow/core/framework/op_gen_lib.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/init_main.h" + +namespace tensorflow { + +namespace { +constexpr char kApiDefFilePattern[] = "*.pbtxt"; + +Status ConvertFilesFromMultiline(const string& input_dir, + const string& output_dir) { + Env* env = Env::Default(); + + const string file_pattern = io::JoinPath(input_dir, kApiDefFilePattern); + std::vector matching_paths; + TF_CHECK_OK(env->GetMatchingPaths(file_pattern, &matching_paths)); + + if (!env->IsDirectory(output_dir).ok()) { + TF_RETURN_IF_ERROR(env->CreateDir(output_dir)); + } + + for (const auto& path : matching_paths) { + string contents; + TF_RETURN_IF_ERROR(tensorflow::ReadFileToString(env, path, &contents)); + contents = tensorflow::PBTxtFromMultiline(contents); + string output_path = io::JoinPath(output_dir, io::Basename(path)); + // Write contents to output_path + TF_RETURN_IF_ERROR( + tensorflow::WriteStringToFile(env, output_path, contents)); + } + return Status::OK(); +} +} // namespace +} // namespace tensorflow + +int main(int argc, char* argv[]) { + tensorflow::port::InitMain(argv[0], &argc, &argv); + + const std::string usage = + "Usage: convert_from_multiline input_dir output_dir"; + if (argc != 3) { + std::cerr << usage << std::endl; + return -1; + } + TF_CHECK_OK(tensorflow::ConvertFilesFromMultiline(argv[1], argv[2])); + return 0; +} diff --git a/tensorflow/tools/benchmark/BUILD b/tensorflow/tools/benchmark/BUILD index 048035f2b1ec4d052fb49ca7a7874ea0f6e96c58..caa6629c491477ffcd108c52d7ce20f1ab95a0a9 100644 --- a/tensorflow/tools/benchmark/BUILD +++ b/tensorflow/tools/benchmark/BUILD @@ -89,3 +89,12 @@ tf_cc_binary( visibility = ["//visibility:public"], deps = [":benchmark_model_lib"], ) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/tools/benchmark/benchmark_model.cc b/tensorflow/tools/benchmark/benchmark_model.cc index f84ae5c7cee52a710bc7595212e29fceb392af11..2d59299da4d313f4bf8c5174480f355c3575fa30 100644 --- a/tensorflow/tools/benchmark/benchmark_model.cc +++ b/tensorflow/tools/benchmark/benchmark_model.cc @@ -230,6 +230,23 @@ Status CalculateFlops(const GraphDef& graph, return Status::OK(); } +void RecordBenchmarkEntry(const string& output_prefix, + const string& benchmark_name, const string& postfix, + int num_runs, double total_time_s, + double throughput = -1.0) { + std::stringstream stream; + stream << benchmark_name; + if (!postfix.empty()) { + stream << "_" << postfix; + } + + TestReporter node_reporter(output_prefix, stream.str()); + TF_QCHECK_OK(node_reporter.Initialize()); + TF_QCHECK_OK( + node_reporter.Benchmark(num_runs, -1.0, total_time_s, throughput)); + TF_QCHECK_OK(node_reporter.Close()); +} + Status RunBenchmark(const std::vector& inputs, const std::vector& outputs, Session* session, StatSummarizer* stats, int64* inference_time_us) { @@ -350,7 +367,7 @@ int Main(int argc, char** argv) { bool show_type = true; bool show_summary = true; bool show_flops = false; - int warmup_runs = 2; + int warmup_runs = 1; std::vector flag_list = { Flag("graph", &graph, "graph file name"), @@ -441,8 +458,14 @@ int Main(int argc, char** argv) { std::unique_ptr session; std::unique_ptr stats; std::unique_ptr graph_def; + + int64 initialization_start_us = Env::Default()->NowMicros(); Status initialize_status = InitializeSession(num_threads, graph, &session, &graph_def); + int64 initialization_end_us = Env::Default()->NowMicros(); + double initialization_time_s = + (initialization_end_us - initialization_start_us) / 1000000.0; + LOG(INFO) << "Initialized session in " << initialization_time_s << "s"; if (!initialize_status.ok()) { return -1; } @@ -587,11 +610,23 @@ int Main(int argc, char** argv) { static_cast(no_stat_wall_time) / (1024 * 1024); // Report the stats. - TestReporter reporter(output_prefix, benchmark_name); - TF_QCHECK_OK(reporter.Initialize()); - TF_QCHECK_OK(reporter.Benchmark(no_stat_num_runs, -1.0, no_stat_wall_time, - throughput)); - TF_QCHECK_OK(reporter.Close()); + RecordBenchmarkEntry(output_prefix, benchmark_name, "", no_stat_num_runs, + no_stat_wall_time, throughput); + + // Session initialization time. + RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-init", 1, + initialization_time_s); + + // First inference time. Note: if warmup_runs is > 1 this will actually be + // an average of all the warmup runs. + RecordBenchmarkEntry(output_prefix, benchmark_name, "meta-first-inference", + warmup_runs, warmup_time_us / 1000000.0); + + // Time from starting to intialize TF to getting the first result back. + // This also assumes that only one warmup run is performed. + RecordBenchmarkEntry( + output_prefix, benchmark_name, "meta-init-plus-first-inference", 1, + initialization_time_s + (warmup_time_us / 1000000.0) / warmup_runs); std::map node_type_map_count; std::map node_type_map_time; @@ -603,17 +638,10 @@ int Main(int argc, char** argv) { &node_type_map_memory, &node_type_map_times_called, &accumulated_us); for (const auto& time : node_type_map_time) { - std::stringstream stream; - stream << benchmark_name << "_" << time.first; - TestReporter node_reporter(output_prefix, stream.str()); - LOG(INFO) << "Outputting: [" << time.first << "]"; - - TF_QCHECK_OK(node_reporter.Initialize()); - TF_QCHECK_OK(node_reporter.Benchmark( - stat_num_runs, -1.0, (time.second * stat_num_runs) / 1000000.0f, - -1.0)); - TF_QCHECK_OK(node_reporter.Close()); + RecordBenchmarkEntry(output_prefix, benchmark_name, time.first, + stat_num_runs, + (time.second * stat_num_runs) / 1000000.0f); } } diff --git a/tensorflow/tools/ci_build/Dockerfile.android b/tensorflow/tools/ci_build/Dockerfile.android index facff476217d2da9d7ebbe8b87bbcdaa49d5e458..99a69d7b43bbc19f0b1e9ee7c741426c6651dfd6 100644 --- a/tensorflow/tools/ci_build/Dockerfile.android +++ b/tensorflow/tools/ci_build/Dockerfile.android @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake index 9013dc012d90ae17ba57815e3cbab829239c6a4c..37ba24d65a2e95833511fa9b3e4044db634a08fd 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cmake +++ b/tensorflow/tools/ci_build/Dockerfile.cmake @@ -14,7 +14,7 @@ # ============================================================================== FROM ubuntu:16.04 -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/Dockerfile.cpu b/tensorflow/tools/ci_build/Dockerfile.cpu index 206108930a170d814466dbaf2f1c8a1c675aefab..57a854a9df738dea5d8560b54765099f32d0ff86 100644 --- a/tensorflow/tools/ci_build/Dockerfile.cpu +++ b/tensorflow/tools/ci_build/Dockerfile.cpu @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu index b914f51918c898199903fb7e55724fe7e79c8318..eb9d0d4dd01c8b39fd108c88d690a2c08efa3760 100644 --- a/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu +++ b/tensorflow/tools/ci_build/Dockerfile.debian.jessie.cpu @@ -1,6 +1,6 @@ FROM debian:jessie -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu b/tensorflow/tools/ci_build/Dockerfile.gpu index 5d18295f68d9114daa1acbd7c11deaad95920f3f..2d46ccb6b17ac3ab3af49c1649074eda8a840331 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu +++ b/tensorflow/tools/ci_build/Dockerfile.gpu @@ -1,6 +1,6 @@ FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu14.04 -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to # /usr/local/cuda diff --git a/tensorflow/tools/ci_build/Dockerfile.gpu_clang b/tensorflow/tools/ci_build/Dockerfile.gpu_clang index c4342d17f5f680e4688b33e54e4322d7650747bc..0ecd8c75e036fc18d37882834ed467d0edb096b1 100644 --- a/tensorflow/tools/ci_build/Dockerfile.gpu_clang +++ b/tensorflow/tools/ci_build/Dockerfile.gpu_clang @@ -1,6 +1,6 @@ FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu14.04 -MAINTAINER Ilya Biryukov +LABEL maintainer="Ilya Biryukov " # In the Ubuntu 14.04 images, cudnn is placed in system paths. Move them to # /usr/local/cuda diff --git a/tensorflow/tools/ci_build/Dockerfile.hadoop b/tensorflow/tools/ci_build/Dockerfile.hadoop index 489493c26e4b4f7384b76af78a3644aba8c697a1..6010aedb339abadd8ee09d50d4eb279c5d3236f8 100644 --- a/tensorflow/tools/ci_build/Dockerfile.hadoop +++ b/tensorflow/tools/ci_build/Dockerfile.hadoop @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -MAINTAINER Jonathan Hseu +LABEL maintainer="Jonathan Hseu " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/Dockerfile.pi b/tensorflow/tools/ci_build/Dockerfile.pi index 2fddd6a2c004800b20062aed6d92eaad004170d2..75ef30d32b0671f770facc1d9f054c03ea253913 100644 --- a/tensorflow/tools/ci_build/Dockerfile.pi +++ b/tensorflow/tools/ci_build/Dockerfile.pi @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/Dockerfile.pi-python3 b/tensorflow/tools/ci_build/Dockerfile.pi-python3 index 18b131ea19f96a5c8844ef1741ed18a7f709439c..b1c648ba3012c4511a587d27177a1323eab10c8c 100644 --- a/tensorflow/tools/ci_build/Dockerfile.pi-python3 +++ b/tensorflow/tools/ci_build/Dockerfile.pi-python3 @@ -1,6 +1,6 @@ FROM ubuntu:14.04 -MAINTAINER Jan Prach +LABEL maintainer="Jan Prach " # Copy and run the install scripts. COPY install/*.sh /install/ diff --git a/tensorflow/tools/ci_build/README.md b/tensorflow/tools/ci_build/README.md index ad83669950f7b284860f84ce87855fe3e3b3e0a9..202fcb9101a42336f5f33022c3b8608e53d83dae 100644 --- a/tensorflow/tools/ci_build/README.md +++ b/tensorflow/tools/ci_build/README.md @@ -1,115 +1,76 @@ # TensorFlow Builds -This directory contains all the files and setup instructions to run all -the important builds and tests. **You can trivially run it yourself!** It also -run continuous integration [ci.tensorflow.org](https://ci.tensorflow.org). - - +This directory contains all the files and setup instructions to run all the +important builds and tests. You can run it yourself! ## Run It Yourself -1. Install [Docker](http://www.docker.com/). Follow instructions - [on the Docker site](https://docs.docker.com/installation/). - - You can run all the jobs **without docker** if you are on mac or on linux - and you just don't want docker. Just install all the dependencies from - [Installing TensorFlow](https://www.tensorflow.org/install/). - Then run any of the one liners below without the - `tensorflow/tools/ci_build/ci_build.sh` in them. - -2. Clone tensorflow repository. - - ```bash - git clone https://github.com/tensorflow/tensorflow.git - ``` - -3. Go to tensorflow directory - - ```bash - cd tensorflow - ``` - -4. Build what you want, for example - - ```bash - tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... - ``` - If you are using the Docker image on Windows or OS X, the Docker VM's default - memory limit may be too low to build TensorFlow. This can result in - strange-looking errors, e.g. the compilation may fail with `gcc: internal - compiler error: Killed (program cc1plus)`. Try increasing the memory limit in - the Docker preferences. - - -## Jobs - -The jobs run by [ci.tensorflow.org](https://ci.tensorflow.org) include following: - -```bash -# Note: You can run the following one-liners yourself if you have Docker. Run -# without `tensorflow/tools/ci_build/ci_build.sh` on mac or linux without Docker. - -# build and run cpu tests -tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow/... +You have two options when running TensorFlow tests locally on your +machine. First, using docker, you can run our Continuous Integration +(CI) scripts on tensorflow devel images. The other option is to install +all TensorFlow dependencies on your machine and run the scripts +natively on your system. -# build and run gpu tests (note if you get unstable results you may be running -# out of gpu memory - if so add "--jobs=1" argument) -tensorflow/tools/ci_build/ci_build.sh GPU bazel test -c opt --config=cuda //tensorflow/... +### Run TensorFlow CI Scripts using Docker -# build pip with gpu support -tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda +1. Install Docker following the [instructions on the docker website](https://docs.docker.com/engine/installation/). -# build and run gpu tests using python 3 -CI_DOCKER_EXTRA_PARAMS="-e CI_BUILD_PYTHON=python3" tensorflow/tools/ci_build/ci_build.sh GPU tensorflow/tools/ci_build/builds/pip.sh GPU -c opt --config=cuda +2. Start a container with one of the devel images here: + https://hub.docker.com/r/tensorflow/tensorflow/tags/. -# build android example app -tensorflow/tools/ci_build/ci_build.sh ANDROID tensorflow/tools/ci_build/builds/android.sh +3. Based on your choice of the image, pick one of the scripts under + https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/ci_build/linux + and run them from the TensorFlow repository root. -# cmake cpu build and test -tensorflow/tools/ci_build/ci_build.sh CPU tensorflow/tools/ci_build/builds/cmake.sh +### Run TensorFlow CI Scripts Natively on your Machine -# run bash inside the container -CI_DOCKER_EXTRA_PARAMS='-it --rm' tensorflow/tools/ci_build/ci_build.sh CPU /bin/bash -``` +1. Follow the instructions at https://www.tensorflow.org/install/install_sources, + but stop when you get to the section "Configure the installation". You do not + need to configure the installation to run the CI scripts. -**Note**: The set of jobs and how they are triggered is still evolving. -There are builds for master branch on cpu, gpu and android. There is a build -for incoming gerrit changes. Gpu tests and benchmark are coming soon. Check -[ci.tensorflow.org](https://ci.tensorflow.org) for current jobs. +2. Pick the appropriate OS and python version you have installed, + and run the script under tensorflow/tools/ci_build/. +## TensorFlow Continuous Integration +To verify that new changes don’t break TensorFlow, we run builds and +tests on either [Jenkins](https://jenkins-ci.org/) or a CI system +internal to Google. -## How Does TensorFlow Continuous Integration Work +We can trigger builds and tests on updates to master or on each pull +request. Contact one of the repository maintainers to trigger builds +on your pull request. -We use [jenkins](https://jenkins-ci.org/) as our continuous integration. -It is running at [ci.tensorflow.org](https://ci.tensorflow.org). -All the jobs are run within [docker](http://www.docker.com/) containers. +### View CI Results -Builds can be triggered by push to master, push a change set or manually. -The build started in jenkins will first pull the git tree. Then jenkins builds -a docker container (using one of those Dockerfile.* files in this directory). -The build itself is run within the container itself. +The Pull Request will show if the change passed or failed the checks. -Source tree lives in jenkins job workspace. Docker container for jenkins -are transient - deleted after the build. Containers build very fast thanks -to docker caching. Individual builds are fast thanks to bazel caching. +From the pull request, click **Show all checks** to see the list of builds +and tests. Click on **Details** to see the results from Jenkins or the internal +CI system. +Results from Jenkins are displayed in the Jenkins UI. For more information, +see the [Jenkins documentation](https://jenkins.io/doc/). +Results from the internal CI system are displayed in the Build Status UI. In +this UI, to see the logs for a failed build: -## Implementation Details +* Click on the **INVOCATION LOG** tab to see the invocation log. -* The ci_build.sh script create and run docker container with all dependencies. - The builds/with_the_same_user together with ci_build.sh creates an environment - which is the same inside the container as it is outside. The same user, group, - path, so that docker symlinks work inside and outside the container. You can - use it for your development. Edit files in your git clone directory. If you - run the ci_build.sh it gets this directory mapped inside the container and - build your tree. +* Click on the **ARTIFACTS** tab to see a list of all artifacts, including logs. -* The unusual `bazel-ci_build-cache` directory is mapped to docker container - performing the build using docker's --volume parameter. This way we cache - bazel output between builds. +* Individual test logs may be available. To see these logs, from the **TARGETS** + tab, click on the failed target. Then, click on the **TARGET LOG** tab to see + its test log. -* The `builds` directory within this folder contains shell scripts to run within - the container. They essentially contains workarounds for current limitations - of bazel. + If you’re looking at target that is sharded or a test that is flaky, then + the build tool divided the target into multiple shards or ran the test + multiple times. Each test log is specific to the shard, run, and attempt. + To see a specific log: + + 1. Click on the log icon that is on the right next to the shard, run, + and attempt number. + + 2. In the grid that appears on the right, click on the specific shard, + run, and attempt to view its log. You can also type the desired shard, + run, or attempt number in the field above its grid. diff --git a/tensorflow/tools/ci_build/builds/run_pip_tests.sh b/tensorflow/tools/ci_build/builds/run_pip_tests.sh index 43d5c5ff3b84e5bfa0f3e4211816e6701405f9a8..29680e6882371d7917b446d01f0640dbdfa1b56f 100755 --- a/tensorflow/tools/ci_build/builds/run_pip_tests.sh +++ b/tensorflow/tools/ci_build/builds/run_pip_tests.sh @@ -78,6 +78,7 @@ ln -s $(pwd)/tensorflow ${PIP_TEST_ROOT}/tensorflow # tests with no_pip_gpu tag. PIP_TEST_FILTER_TAG="-no_pip,-no_oss" if [[ ${IS_OSS_SERIAL} == "1" ]]; then + PIP_TEST_FILTER_TAG="$(echo "${PIP_TEST_FILTER_TAG}" | sed s/-no_oss//)" PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},oss_serial" else PIP_TEST_FILTER_TAG="${PIP_TEST_FILTER_TAG},-oss_serial" diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh index 4e72d025a22cb90428e396c0cfdd1a7c545222eb..f1c207f9b686a77d92f2df52faaf7da4f55c5d31 100755 --- a/tensorflow/tools/ci_build/ci_sanity.sh +++ b/tensorflow/tools/ci_build/ci_sanity.sh @@ -95,6 +95,8 @@ do_pylint() { "^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined "\ "^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated "\ "^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated "\ +"^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden "\ +"^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden "\ "^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator "\ "^tensorflow/python/keras/_impl/keras/callbacks\.py.*\[E1133.*not-an-iterable" @@ -424,6 +426,72 @@ do_code_link_check() { tensorflow/tools/ci_build/code_link_check.sh } +# List .h|.cc files changed in the last non-merge git commit that still exist, +# i.e., not removed. +# Usage: get_clang_files_to_check [--incremental] +get_clang_files_to_check() { + if [[ "$1" == "--incremental" ]]; then + CHANGED_CLANG_FILES=$(get_changed_files_in_last_non_merge_git_commit | \ + grep '.*\.h$\|.*\.cc$') + + # Do not include files removed in the last non-merge commit. + CLANG_FILES="" + for CLANG_FILE in ${CHANGED_CLANG_FILES}; do + if [[ -f "${CLANG_FILE}" ]]; then + CLANG_FILES="${CLANG_FILES} ${CLANG_FILE}" + fi + done + + echo "${CLANG_FILES}" + else + find tensorflow -name '*.h' -o -name '*.cc' + fi +} + +do_clang_format_check() { + if [[ $# != "0" ]] && [[ $# != "1" ]]; then + echo "Invalid syntax when invoking do_clang_format_check" + echo "Usage: do_clang_format_check [--incremental]" + return 1 + fi + + if [[ "$1" == "--incremental" ]]; then + CLANG_SRC_FILES=$(get_clang_files_to_check --incremental) + + if [[ -z "${CLANG_SRC_FILES}" ]]; then + echo "do_clang_format_check will NOT run due to --incremental flag and "\ +"due to the absence of .h or .cc code changes in the last commit." + return 0 + fi + elif [[ -z "$1" ]]; then + # TODO (yongtang): Always pass --incremental until all files have + # been sanitized gradually. Then this --incremental could be removed. + CLANG_SRC_FILES=$(get_clang_files_to_check --incremental) + else + echo "Invalid syntax for invoking do_clang_format_check" + echo "Usage: do_clang_format_check [--incremental]" + return 1 + fi + + CLANG_FORMAT=${CLANG_FORMAT:-clang-format-3.8} + + success=1 + for filename in $CLANG_SRC_FILES; do + $CLANG_FORMAT --style=google $filename | diff $filename - > /dev/null + if [ ! $? -eq 0 ]; then + success=0 + echo File $filename is not properly formatted with "clang-format "\ +"--style=google" + fi + done + + if [ $success == 0 ]; then + echo Clang format check fails. + exit 1 + fi + echo Clang format check success. +} + do_check_load_py_test() { BUILD_CMD="bazel build ${BAZEL_FLAGS} //tensorflow/tools/pip_package:check_load_py_test" ${BUILD_CMD} diff --git a/tensorflow/tools/ci_build/install/build_and_install_clang.sh b/tensorflow/tools/ci_build/install/build_and_install_clang.sh index 3fb99649485ef1719c5c3b561f21a21b49844c91..99664344777256b9eb8c3764bb1900f26b43cc6e 100755 --- a/tensorflow/tools/ci_build/install/build_and_install_clang.sh +++ b/tensorflow/tools/ci_build/install/build_and_install_clang.sh @@ -16,7 +16,7 @@ set -ex -LLVM_SVN_REVISION="299268" +LLVM_SVN_REVISION="314281" CLANG_TMP_DIR=/tmp/clang-build mkdir "$CLANG_TMP_DIR" diff --git a/tensorflow/tools/ci_build/install/install_deb_packages.sh b/tensorflow/tools/ci_build/install/install_deb_packages.sh index da1f2199d0daf5cfe3e9d94165e3af6704c58050..4ab307c9253a8019f2c794b696db030722751770 100755 --- a/tensorflow/tools/ci_build/install/install_deb_packages.sh +++ b/tensorflow/tools/ci_build/install/install_deb_packages.sh @@ -28,6 +28,7 @@ if [[ "$1" != "" ]] && [[ "$1" != "--without_cmake" ]]; then fi # Install dependencies from ubuntu deb repository. +apt-key adv --keyserver keyserver.ubuntu.com --recv 084ECFC5828AB726 apt-get update if [[ "$ubuntu_version" == "14" ]]; then @@ -41,6 +42,7 @@ apt-get install -y --no-install-recommends \ autoconf \ automake \ build-essential \ + clang-format-3.8 \ curl \ ffmpeg \ git \ diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh new file mode 100755 index 0000000000000000000000000000000000000000..8f839ca110e5bbeba6fb7f0baaeab2fe6f126319 --- /dev/null +++ b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +set -e +set -x + +N_JOBS=$(sysctl -n hw.ncpu) +N_JOBS=$((N_JOBS+1)) + +echo "" +echo "Bazel will use ${N_JOBS} concurrent job(s)." +echo "" + +# Run configure. +export TF_NEED_CUDA=0 +export PYTHON_BIN_PATH=$(which python3) +yes "" | $PYTHON_BIN_PATH configure.py +which bazel +bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac \ + --test_timeout 300,450,1200,3600 \ + --test_size_filters=small,medium \ + --jobs=${N_JOBS} --build_tests_only --test_output=errors -k -- \ + //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/... diff --git a/tensorflow/tools/ci_build/update_version.py b/tensorflow/tools/ci_build/update_version.py index c7841f35aacf0580423cbfd184f5e882f08336ec..d2a63e5d66a34f61d17e8327d4b25320371c4fa3 100755 --- a/tensorflow/tools/ci_build/update_version.py +++ b/tensorflow/tools/ci_build/update_version.py @@ -17,7 +17,7 @@ # Automatically update TensorFlow version in source files # # Usage: -# ./tensorflow/tools/ci_build/update_version.py --version 1.4.0-rc0 +# ./tensorflow/tools/ci_build/update_version.py --version 1.4.0-rc1 # ./tensorflow/tools/ci_build/update_version.py --nightly # """Update version of TensorFlow script.""" diff --git a/tensorflow/tools/dist_test/Dockerfile b/tensorflow/tools/dist_test/Dockerfile index cd64e2c518ed8f602a0b32a7d6092ebc67e0d4f8..2a7605bbc960f1caccd6163fb5867639c48fa70c 100644 --- a/tensorflow/tools/dist_test/Dockerfile +++ b/tensorflow/tools/dist_test/Dockerfile @@ -20,7 +20,7 @@ FROM ubuntu:16.04 -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " RUN apt-get update RUN apt-get install -y \ diff --git a/tensorflow/tools/dist_test/Dockerfile.local b/tensorflow/tools/dist_test/Dockerfile.local index 7a896ab611ad68fc7c6621a8a4e1dc9b72b15516..795aeee1b5d21f973dfa5856969ef3a85d2571ca 100644 --- a/tensorflow/tools/dist_test/Dockerfile.local +++ b/tensorflow/tools/dist_test/Dockerfile.local @@ -19,7 +19,7 @@ FROM ubuntu:16.04 -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " # Pick up some TF dependencies. RUN apt-get update && apt-get install -y \ diff --git a/tensorflow/tools/dist_test/local/Dockerfile b/tensorflow/tools/dist_test/local/Dockerfile index 96846f656489e0f2349839ec2768648d4eeef8c2..383c3c2f4ca426b7e73ec074a452bdb3125c2efb 100644 --- a/tensorflow/tools/dist_test/local/Dockerfile +++ b/tensorflow/tools/dist_test/local/Dockerfile @@ -1,6 +1,6 @@ FROM jpetazzo/dind -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " RUN apt-get update diff --git a/tensorflow/tools/dist_test/server/Dockerfile b/tensorflow/tools/dist_test/server/Dockerfile index fabc8a7105e17a6b3d7ca1bcbc356cfc54eba362..1359428f1140b6fd6ecf3b14fc5b968b49d4576a 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile +++ b/tensorflow/tools/dist_test/server/Dockerfile @@ -19,7 +19,7 @@ FROM ubuntu:16.04 -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " # Pick up some TF dependencies RUN apt-get update && apt-get install -y \ diff --git a/tensorflow/tools/dist_test/server/Dockerfile.test b/tensorflow/tools/dist_test/server/Dockerfile.test index 908af8af9bb0cc67ae21aec72b42faab970093cc..ce7e783a1a846db175d9da9ff66572452c3573cd 100644 --- a/tensorflow/tools/dist_test/server/Dockerfile.test +++ b/tensorflow/tools/dist_test/server/Dockerfile.test @@ -19,7 +19,7 @@ FROM ubuntu:16.04 -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " # Pick up some TF dependencies RUN apt-get update && apt-get install -y \ diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 index 75351ecfba6975332e4a1e66e506c8ea16c138e8..64ebc4607a82ce59bd3e13c28541ca93778ecdb7 100644 --- a/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 +++ b/tensorflow/tools/docker/Dockerfile.devel-gpu-cuda9-cudnn7 @@ -1,6 +1,6 @@ FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04 -MAINTAINER Gunhan Gulsoy +LABEL maintainer="Gunhan Gulsoy " # It is possible to override these for releases. ARG TF_BRANCH=master @@ -42,6 +42,7 @@ RUN pip --no-cache-dir install \ scipy \ sklearn \ pandas \ + wheel \ && \ python -m ipykernel.kernelspec @@ -80,22 +81,32 @@ RUN git clone https://github.com/tensorflow/tensorflow.git && \ WORKDIR /tensorflow # Configure the build for our CUDA configuration. -ENV CI_BUILD_PYTHON python -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH -ENV TF_NEED_CUDA 1 -ENV TF_CUDA_COMPUTE_CAPABILITIES 3.0,3.5,5.2,6.0,6.1 -ENV TF_CUDA_VERSION 9.0 -ENV TF_CUDNN_VERSION 7 +ENV CI_BUILD_PYTHON=python \ + LD_LIBRARY_PATH=/usr/local/cuda/extras/CUPTI/lib64:${LD_LIBRARY_PATH} \ + CUDNN_INSTALL_PATH=/usr/lib/x86_64-linux-gnu \ + PYTHON_BIN_PATH=/usr/bin/python \ + PYTHON_LIB_PATH=/usr/local/lib/python2.7/dist-packages \ + TF_NEED_CUDA=1 \ + TF_CUDA_VERSION=9.0 \ + TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1,7.0 \ + TF_CUDNN_VERSION=7 RUN ./configure -RUN LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ - bazel build -c opt --config=cuda --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ - --jobs=${TF_AVAILABLE_CPUS} \ - tensorflow/tools/pip_package:build_pip_package && \ - mkdir -p /pip_pkg && \ +# Build and Install TensorFlow. +RUN ln -s /usr/local/cuda/lib64/stubs/libcuda.so /usr/local/cuda/lib64/stubs/libcuda.so.1 && \ + LD_LIBRARY_PATH=/usr/local/cuda/lib64/stubs:${LD_LIBRARY_PATH} \ + bazel build -c opt \ + --config=cuda \ + --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" \ + --jobs=${TF_AVAILABLE_CPUS} \ + tensorflow/tools/pip_package:build_pip_package && \ + mkdir /pip_pkg && \ bazel-bin/tensorflow/tools/pip_package/build_pip_package /pip_pkg +# Clean up pip wheel and Bazel cache when done. RUN pip --no-cache-dir install --upgrade /pip_pkg/tensorflow-*.whl && \ + rm -rf /pip_pkg && \ + rm -rf /root/.cache WORKDIR /root diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py index 1015103077892e195f93e6bcc62699477c23c36f..3db164c2b5b78dbcb3c408ce89c067d33c2a2af4 100644 --- a/tensorflow/tools/docs/parser.py +++ b/tensorflow/tools/docs/parser.py @@ -107,23 +107,40 @@ def _get_raw_docstring(py_object): # A regular expression for capturing a @{symbol} reference. -SYMBOL_REFERENCE_RE = re.compile(r'@\{([^}]+)\}') +SYMBOL_REFERENCE_RE = re.compile( + r""" + # Start with a literal "@{". + @\{ + # Group at least 1 symbol: not "}" or "\n". + ([^}\n]+) + # Followed by a closing "}" + \} + """, + flags=re.VERBOSE) class ReferenceResolver(object): """Class for replacing @{...} references with Markdown links. - Args: - duplicate_of: A map from duplicate names to preferred names of API - symbols. - doc_index: A `dict` mapping symbol name strings to objects with `url` - and `title` fields. Used to resolve @{$doc} references in docstrings. - index: A map from all full names to python objects. - py_module_names: A list of string names of Python modules. + Attributes: + current_doc_full_name: A string (or None) indicating the name of the + document currently being processed, so errors can reference the broken + doc. """ def __init__(self, duplicate_of, doc_index, is_class, is_module, py_module_names): + """Initializes a Reference Resolver. + + Args: + duplicate_of: A map from duplicate names to preferred names of API + symbols. + doc_index: A `dict` mapping symbol name strings to objects with `url` + and `title` fields. Used to resolve @{$doc} references in docstrings. + is_class: A map from full names to bool for each symbol. + is_module: A map from full names to bool for each symbol. + py_module_names: A list of string names of Python modules. + """ self._duplicate_of = duplicate_of self._doc_index = doc_index self._is_class = is_class @@ -249,11 +266,19 @@ class ReferenceResolver(object): Returns: A markdown link to the documentation page of `ref_full_name`. """ - link = self.reference_to_url(ref_full_name, relative_path_to_root) + url = self.reference_to_url(ref_full_name, relative_path_to_root) + if code_ref: - return '[`%s`](%s)' % (link_text, link) + link_text = link_text.join(['', '']) else: - return '[%s](%s)' % (link_text, link) + link_text = self._link_text_to_html(link_text) + + return '{}'.format(url, link_text) + + @staticmethod + def _link_text_to_html(link_text): + code_re = '`(.*?)`' + return re.sub(code_re, r'\1', link_text) def py_master_name(self, full_name): """Return the master name for a Python symbol name.""" @@ -322,13 +347,13 @@ class ReferenceResolver(object): # Handle different types of references. if string.startswith('$'): # Doc reference - return self._doc_link( - string, link_text, manual_link_text, relative_path_to_root) + return self._doc_link(string, link_text, manual_link_text, + relative_path_to_root) elif string.startswith('tensorflow::'): # C++ symbol - return self._cc_link( - string, link_text, manual_link_text, relative_path_to_root) + return self._cc_link(string, link_text, manual_link_text, + relative_path_to_root) else: is_python = False @@ -337,8 +362,11 @@ class ReferenceResolver(object): is_python = True break if is_python: # Python symbol - return self.python_link(link_text, string, relative_path_to_root, - code_ref=not manual_link_text) + return self.python_link( + link_text, + string, + relative_path_to_root, + code_ref=not manual_link_text) # Error! self.add_error('Did not understand "%s"' % match.group(0)) @@ -361,7 +389,9 @@ class ReferenceResolver(object): if not manual_link_text: link_text = self._doc_index[string].title url = os.path.normpath(os.path.join( relative_path_to_root, '../..', self._doc_index[string].url)) - return '[%s](%s%s)' % (link_text, url, hash_tag) + link_text = self._link_text_to_html(link_text) + return '{}'.format(url, hash_tag, link_text) + return self._doc_missing(string, hash_tag, link_text, manual_link_text, relative_path_to_root) @@ -392,7 +422,9 @@ class ReferenceResolver(object): # to api_docs/cc, and then add ret. cc_relative_path = os.path.normpath(os.path.join( relative_path_to_root, '../cc', ret)) - return '[`%s`](%s)' % (link_text, cc_relative_path) + + return '{}'.format(cc_relative_path, + link_text) # TODO(aselle): Collect these into a big list for all modules and functions diff --git a/tensorflow/tools/docs/parser_test.py b/tensorflow/tools/docs/parser_test.py index 3b74a13f08b9747182240e31d91716badc0ef5ed..8a0e9af5216c881326449b3e85b94c0be331fa37 100644 --- a/tensorflow/tools/docs/parser_test.py +++ b/tensorflow/tools/docs/parser_test.py @@ -75,8 +75,9 @@ class ParserTest(googletest.TestCase): def foo(self): pass - string = ('A @{tf.reference}, another @{tf.reference}, ' - 'a member @{tf.reference.foo}, and a @{tf.third}.') + string = ( + 'A @{tf.reference}, another @{tf.reference}, a member ' + '@{tf.reference.foo}, and a @{tf.third$link `text` with `code` in it}.') duplicate_of = {'tf.third': 'tf.fourth'} index = {'tf.reference': HasOneMember, 'tf.reference.foo': HasOneMember.foo, @@ -89,12 +90,15 @@ class ParserTest(googletest.TestCase): visitor=visitor, doc_index={}, py_module_names=['tf']) result = reference_resolver.replace_references(string, '../..') - self.assertEqual( - 'A [`tf.reference`](../../tf/reference.md), another ' - '[`tf.reference`](../../tf/reference.md), ' - 'a member [`tf.reference.foo`](../../tf/reference.md#foo), ' - 'and a [`tf.third`](../../tf/fourth.md).', - result) + self.assertEqual('A ' + 'tf.reference, ' + 'another ' + 'tf.reference, ' + 'a member ' + 'tf.reference.foo, ' + 'and a link ' + 'text with ' + 'code in it.', result) def test_doc_replace_references(self): string = '@{$doc1} @{$doc1#abc} @{$doc1$link} @{$doc1#def$zelda} @{$do/c2}' @@ -114,10 +118,11 @@ class ParserTest(googletest.TestCase): reference_resolver = parser.ReferenceResolver.from_visitor( visitor=visitor, doc_index=doc_index, py_module_names=['tf']) result = reference_resolver.replace_references(string, 'python') - self.assertEqual( - '[Title1](../URL1) [Title1](../URL1#abc) [link](../URL1) ' - '[zelda](../URL1#def) [Two words](../somewhere/else)', - result) + self.assertEqual('Title1 ' + 'Title1 ' + 'link ' + 'zelda ' + 'Two words', result) def test_docs_for_class(self): @@ -389,7 +394,7 @@ class ParserTest(googletest.TestCase): self.assertIn('TestModule.test_function', docs) # Leading backtick to make sure it's included top-level. # This depends on formatting, but should be stable. - self.assertIn('`test_function', docs) + self.assertIn('test_function', docs) def test_argspec_for_functools_partial(self): diff --git a/tensorflow/tools/docs/pretty_docs.py b/tensorflow/tools/docs/pretty_docs.py index 5ea93948657e87e73d5bc85344bb5d81a26a1d9d..c033c16ae98c4bcaa4c0338e539324b3a2ae5552 100644 --- a/tensorflow/tools/docs/pretty_docs.py +++ b/tensorflow/tools/docs/pretty_docs.py @@ -117,7 +117,8 @@ def _build_class_page(page_info): parts.append(page_info.guides) parts.append(page_info.doc.docstring) parts.append(_build_function_details(page_info.doc.function_details)) - assert not page_info.doc.compatibility + parts.append(_build_compatibility(page_info.doc.compatibility)) + parts.append('\n\n') if page_info.classes: @@ -139,7 +140,8 @@ def _build_class_page(page_info): parts.append(prop_info.doc.docstring) parts.append(_build_function_details(prop_info.doc.function_details)) - assert not prop_info.doc.compatibility + parts.append(_build_compatibility(prop_info.doc.compatibility)) + parts.append('\n\n') parts.append('\n\n') @@ -206,6 +208,8 @@ def _build_module_page(page_info): parts.append(str(page_info.defined_in)) parts.append(page_info.doc.docstring) + parts.append(_build_compatibility(page_info.doc.compatibility)) + parts.append('\n\n') if page_info.modules: @@ -290,7 +294,9 @@ def _build_compatibility(compatibility): for key in sorted_keys: value = compatibility[key] - parts.append('\n\n#### %s compatibility\n%s\n' % (key, value)) + # Dedent so that it does not trigger markdown code formatting. + value = textwrap.dedent(value) + parts.append('\n\n#### %s Compatibility\n%s\n' % (key.title(), value)) return ''.join(parts) diff --git a/tensorflow/tools/gcs_test/Dockerfile b/tensorflow/tools/gcs_test/Dockerfile index 5af753226fc91d74ade56d59316aab5a004a15b7..69b554047bb8ea3b929e111de16285c20099a55f 100644 --- a/tensorflow/tools/gcs_test/Dockerfile +++ b/tensorflow/tools/gcs_test/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:16.04 -MAINTAINER Shanqing Cai +LABEL maintainer="Shanqing Cai " RUN apt-get update RUN apt-get install -y \ diff --git a/tensorflow/tools/gcs_test/python/gcs_smoke.py b/tensorflow/tools/gcs_test/python/gcs_smoke.py index 9882f75a8ad610d1faf7bd97c70a05a7e133b445..ad4cb17ae1d16bc0d7469fb43ec26b96a41ea68c 100644 --- a/tensorflow/tools/gcs_test/python/gcs_smoke.py +++ b/tensorflow/tools/gcs_test/python/gcs_smoke.py @@ -35,6 +35,7 @@ flags.DEFINE_integer("num_examples", 10, "Number of examples to generate") FLAGS = flags.FLAGS + def create_examples(num_examples, input_mean): """Create ExampleProto's containing data.""" ids = np.arange(num_examples).reshape([num_examples, 1]) @@ -49,6 +50,7 @@ def create_examples(num_examples, input_mean): examples.append(ex) return examples + def create_dir_test(): """Verifies file_io directory handling methods.""" @@ -122,6 +124,7 @@ def create_dir_test(): print("Deleted directory recursively %s in %s milliseconds" % ( dir_name, elapsed_ms)) + def create_object_test(): """Verifies file_io's object manipulation methods .""" starttime_ms = int(round(time.time() * 1000)) @@ -142,7 +145,8 @@ def create_object_test(): print("Creating file %s." % file_name) file_io.write_string_to_file(file_name, "test file creation.") elapsed_ms = int(round(time.time() * 1000)) - starttime_ms - print("Created %d files in %s milliseconds" % (len(files_to_create), elapsed_ms)) + print("Created %d files in %s milliseconds" % ( + len(files_to_create), elapsed_ms)) # Listing files of pattern1. list_files_pattern = "%s/test_file*.txt" % dir_name @@ -185,7 +189,9 @@ def create_object_test(): file_io.delete_recursively(dir_name) -if __name__ == "__main__": +def main(argv): + del argv # Unused. + # Sanity check on the GCS bucket URL. if not FLAGS.gcs_bucket_url or not FLAGS.gcs_bucket_url.startswith("gs://"): print("ERROR: Invalid GCS bucket URL: \"%s\"" % FLAGS.gcs_bucket_url) @@ -210,7 +216,7 @@ if __name__ == "__main__": # tf_record_iterator works. record_iter = tf.python_io.tf_record_iterator(input_path) read_count = 0 - for r in record_iter: + for _ in record_iter: read_count += 1 print("Read %d records using tf_record_iterator" % read_count) @@ -222,7 +228,7 @@ if __name__ == "__main__": # Verify that running the read op in a session works. print("\n=== Testing TFRecordReader.read op in a session... ===") - with tf.Graph().as_default() as g: + with tf.Graph().as_default(): filename_queue = tf.train.string_input_producer([input_path], num_epochs=1) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) @@ -249,3 +255,7 @@ if __name__ == "__main__": create_dir_test() create_object_test() + + +if __name__ == "__main__": + tf.app.run(main) diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py index a7f8b5bb5f265892a3d11d63569c7044ae440af7..616ec9fbe0251f9b3d3e7d6f788c193f7856006d 100755 --- a/tensorflow/tools/git/gen_git_source.py +++ b/tensorflow/tools/git/gen_git_source.py @@ -170,8 +170,16 @@ def write_version_info(filename, git_version): if b"\"" in git_version or b"\\" in git_version: git_version = "git_version_is_invalid" # do not cause build to fail! contents = """/* Generated by gen_git_source.py */ +#include const char* tf_git_version() {return "%s";} const char* tf_compiler_version() {return __VERSION__;} +const int tf_cxx11_abi_flag() { +#ifdef _GLIBCXX_USE_CXX11_ABI + return _GLIBCXX_USE_CXX11_ABI; +#else + return -1; +#endif +} """ % git_version open(filename, "w").write(contents) diff --git a/tensorflow/tools/git/gen_git_source.sh b/tensorflow/tools/git/gen_git_source.sh index 977fe16333d573f61a4f282d5839b4b60507b479..eb5e1abe15eb8be0f0580a8b7412f2db6fbea616 100755 --- a/tensorflow/tools/git/gen_git_source.sh +++ b/tensorflow/tools/git/gen_git_source.sh @@ -26,7 +26,15 @@ if [[ $? != 0 ]]; then fi cat < ${OUTPUT_FILENAME} +#include const char* tf_git_version() {return "${GIT_VERSION}";} const char* tf_compiler_version() {return __VERSION__;} +const int tf_cxx11_abi_flag() { +#ifdef _GLIBCXX_USE_CXX11_ABI + return _GLIBCXX_USE_CXX11_ABI; +#else + return -1; +#endif +} EOF diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc index 30290c7a16989177749b05ae6b0b902b8f514a95..f2934a79bdf65473092cbf80fafbda888d7b9c7c 100644 --- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc +++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc @@ -17,12 +17,20 @@ limitations under the License. #include #include +#include +#include +#include +#include +#include +#include #include "tensorflow/core/common_runtime/constant_folding.h" #include "tensorflow/core/common_runtime/shape_refiner.h" #include "tensorflow/core/graph/graph_constructor.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/graph/subgraph.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/util/command_line_flags.h" @@ -30,33 +38,38 @@ limitations under the License. namespace tensorflow { namespace graph_transforms { +namespace { +using StringPieceSet = std::unordered_set; +template +using StringPieceMap = std::unordered_map; +} // namespace Status ReplaceSendRecvs(const GraphDef& original_graph_def, const GraphDef& rewritten_graph_def, const std::vector& inputs, const std::vector& outputs, GraphDef* output_graph_def) { - std::map original_map; - MapNamesToNodes(original_graph_def, &original_map); - std::map new_node_names; - for (const NodeDef& node : rewritten_graph_def.node()) { - // If the op isn't a Recv, or it was in the original, nothing to do. - if ((node.op() != "_Recv") || (original_map.count(node.name()) == 1)) { - continue; - } - // See if it matches an input from the original. - for (const string& input : inputs) { - // Here we rely on the naming convention for the Recv nodes that - // RewriteGraphForExecution adds in the place of the feed inputs. - string input_prefix = "_recv_" + input + "_"; - if (StringPiece(node.name()).starts_with(input_prefix)) { - // If it does, prepare to rename any inputs that refer to it. - new_node_names[node.name()] = input; - } - } + // recv_node_names serves as a string storage for recv node names. + std::vector recv_node_names(inputs.size()); + StringPieceMap recv_node_map; + StringPieceSet input_nodes; + for (int i = 0; i < inputs.size(); ++i) { + // RewriteGraphForExecution adds a recv node for each input edge. We assume + // here that adding such recv node did not fail. For example, the original + // graph did not already have a node with the name for the new added recv + // node. + TensorId id = ParseTensorName(inputs[i]); + input_nodes.insert(id.first); + string& recv_node_name = recv_node_names[i]; + recv_node_name = strings::StrCat("_recv_", id.first, "_", id.second); + recv_node_map.emplace(recv_node_name, id); + } + + StringPieceMap original_map; + for (const NodeDef& node : original_graph_def.node()) { + original_map.emplace(node.name(), &node); } - std::vector nodes_to_add; for (const NodeDef& node : rewritten_graph_def.node()) { if ((node.op() == "_Send") || (node.op() == "_Recv")) { // If the op is a Send or Recv that wasn't in the original, skip it. @@ -64,55 +77,68 @@ Status ReplaceSendRecvs(const GraphDef& original_graph_def, continue; } } - NodeDef new_node; - new_node = node; - new_node.mutable_input()->Clear(); - for (const string& old_input : node.input()) { - string input_prefix; - string input_node_name; - string input_suffix; - NodeNamePartsFromInput(old_input, &input_prefix, &input_node_name, - &input_suffix); - string new_input; - if (new_node_names.count(input_node_name) > 0) { - new_input = - input_prefix + new_node_names[input_node_name] + input_suffix; - } else { - new_input = old_input; + + NodeDef* new_node = output_graph_def->add_node(); + new_node->MergeFrom(node); + for (int i = 0; i < new_node->input_size(); ++i) { + string& input = *new_node->mutable_input(i); + TensorId id = ParseTensorName(input); + const auto iter = recv_node_map.find(id.first); + if (iter != recv_node_map.end()) { + // The node being substituted is a Recv node, and it has only one + // output. If this input is not a control input, then replace the input + // with the mapped value. Otherwise, replace the node name only. + if (id.second != Graph::kControlSlot) { + CHECK_EQ(id.second, 0); + input = iter->second.ToString(); + } else { + id.first = iter->second.first; + input = id.ToString(); + } } - *(new_node.mutable_input()->Add()) = new_input; } - nodes_to_add.push_back(new_node); - } - for (std::pair entry : new_node_names) { - string removed_node_name = entry.second; - const NodeDef* removed_node = original_map[removed_node_name]; - NodeDef new_node; - new_node = *removed_node; - nodes_to_add.push_back(new_node); + + // RewriteGraphForExecution() did not remove this input node. Remove this + // node name from input_nodes so that a duplicate does not get added to the + // output_graph_def. + auto iter = input_nodes.find(new_node->name()); + if (iter != input_nodes.end()) { + input_nodes.erase(iter); + } } - for (const NodeDef& node : nodes_to_add) { - *output_graph_def->mutable_node()->Add() = node; + // Some input nodes are removed in rewrite_graph_def. Add those nodes to + // output_graph_def. + for (StringPiece name : input_nodes) { + const NodeDef& removed_node = *CHECK_NOTNULL(original_map[name]); + output_graph_def->add_node()->MergeFrom(removed_node); } + return Status::OK(); } Status RemoveUnusedNodes(const GraphDef& input_graph_def, const TransformFuncContext& context, GraphDef* output_graph_def) { - std::map node_map; - MapNamesToNodes(input_graph_def, &node_map); + StringPieceMap node_map; + for (const NodeDef& node : input_graph_def.node()) { + node_map.emplace(node.name(), &node); + } - std::set used_nodes; + std::unordered_set input_names; for (const string& input : context.input_names) { - used_nodes.insert(input); + input_names.insert(ParseTensorName(input)); + } + StringPieceSet used_nodes; + StringPieceSet current_nodes; + for (const string& name : context.output_names) { + TensorId id = ParseTensorName(name); + used_nodes.insert(id.first); + current_nodes.insert(id.first); } - std::vector current_nodes = context.output_names; while (!current_nodes.empty()) { - std::set next_nodes; - for (const string& node_name : current_nodes) { - used_nodes.insert(node_name); + StringPieceSet next_nodes; + for (StringPiece node_name : current_nodes) { if (node_map.count(node_name) == 0) { LOG(ERROR) << "Bad graph structure, no node named '" << node_name << "' found for input lookup"; @@ -120,14 +146,20 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def, node_name, "' found for input lookup"); } const NodeDef& node = *(node_map[node_name]); - for (const string& input_name : node.input()) { - const string& input_node_name = NodeNameFromInput(input_name); - if (used_nodes.count(input_node_name) == 0) { - next_nodes.insert(input_node_name); + for (const string& input : node.input()) { + TensorId id = ParseTensorName(input); + if (input_names.count(id) > 0) { + continue; + } + if (used_nodes.insert(id.first).second) { + next_nodes.insert(id.first); } } } - current_nodes = std::vector(next_nodes.begin(), next_nodes.end()); + current_nodes.swap(next_nodes); + } + for (const TensorId& id : input_names) { + used_nodes.insert(id.first); } FilterGraphDef( input_graph_def, @@ -141,7 +173,7 @@ Status RemoveUnusedNodes(const GraphDef& input_graph_def, Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle, shape_inference::InferenceContext* context, PartialTensorShape* shape) { - // The default is already unknown + // The default is already unknown. if (!context->RankKnown(handle)) return Status::OK(); std::vector dims(context->Rank(handle)); @@ -151,47 +183,6 @@ Status ShapeHandleToTensorShape(const shape_inference::ShapeHandle& handle, return PartialTensorShape::MakePartialShape(dims.data(), dims.size(), shape); } -Status ShapeForNode(const TransformFuncContext& context, - const string& node_name, TensorShape* result, - bool* has_shape_specified) { - *has_shape_specified = false; - - // Check to see if we have been given a default for all placeholders. - if (context.params.count("type")) { - if (context.params.at("shape").size() != 1) { - return errors::InvalidArgument( - "You must pass no more than one default 'shape' to " - "fold_constants"); - } - const string& shape_string = context.params.at("shape")[0]; - TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result)); - *has_shape_specified = true; - } - - // See if there's a particular type specified for this placeholder. - if (context.params.count("name") || context.params.count("type_for_name")) { - if (!context.params.count("name") || - !context.params.count("type_for_name") || - (context.params.at("type_for_name").size() != - context.params.at("name").size())) { - return errors::InvalidArgument( - "You must pass a 'shape_for_name' arg for every 'name', e.g. " - "fold_constants(name=foo, shape_for_name=\"2,2,1\", name=bar, " - "shape_for_name=\"1\""); - } - const int name_count = context.params.at("name").size(); - for (int i = 0; i < name_count; ++i) { - if (context.params.at("name")[i] == node_name) { - const string& shape_string = context.params.at("shape_for_name")[i]; - TF_RETURN_IF_ERROR(TensorShapeFromString(shape_string, result)); - *has_shape_specified = true; - } - } - } - - return Status::OK(); -} - // Converts any sub-graphs that can be resolved into constant expressions into // single Const ops. Status FoldConstants(const GraphDef& input_graph_def, @@ -215,17 +206,6 @@ Status FoldConstants(const GraphDef& input_graph_def, GraphDef cleaned_graph_def; RemoveAttributes(input_graph_def, {"_output_shapes"}, &cleaned_graph_def); - // Set specified shapes. - for (NodeDef& node : *cleaned_graph_def.mutable_node()) { - TensorShape shape; - bool has_shape_specified; - TF_RETURN_IF_ERROR( - ShapeForNode(context, node.name(), &shape, &has_shape_specified)); - if (has_shape_specified) { - SetNodeAttr("shape", shape, &node); - } - } - TF_RETURN_IF_ERROR( ImportGraphDef({}, cleaned_graph_def, &input_graph, &shape_refiner)); } else { diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc index fd4188a6a4bebbb8079e55612406938b8835b974..41106de008d832a022290e6da38cca8ad6d23ffd 100644 --- a/tensorflow/tools/graph_transforms/fold_constants_test.cc +++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc @@ -74,6 +74,9 @@ class ConstantFoldingTest : public ::testing::Test { TestConstantFolding(graph_def, {{"placeholder_expect_remains", placeholder_tensor}}, {}, {"output_expect_remains"}, {}); + TestConstantFolding(graph_def, + {{"placeholder_expect_remains:0", placeholder_tensor}}, + {}, {"output_expect_remains:0"}, {}); } void TestOpExclusionAdd() { @@ -256,10 +259,40 @@ class ConstantFoldingTest : public ::testing::Test { EXPECT_EQ(0, node_map.count("new_send")); } + void TestReplaceSendRecvsPrefixNames() { + using namespace ::tensorflow::ops; // NOLINT(build/namespaces) + + auto o_root = tensorflow::Scope::NewRootScope(); + auto a = Placeholder(o_root.WithOpName("placeholder"), DT_FLOAT); + auto b = Placeholder(o_root.WithOpName("placeholder_1"), DT_FLOAT); + auto add_o = Add(o_root.WithOpName("add"), a, b); + GraphDef o_graph_def; + TF_ASSERT_OK(o_root.ToGraphDef(&o_graph_def)); + + auto n_root = tensorflow::Scope::NewRootScope(); + auto c = _Recv(n_root.WithOpName("_recv_placeholder_0"), DT_FLOAT, "", "", + 0, ""); + auto d = _Recv(n_root.WithOpName("_recv_placeholder_1_0"), DT_FLOAT, "", "", + 0, ""); + auto add_n = Add(n_root.WithOpName("add"), c, d); + GraphDef n_graph_def; + TF_ASSERT_OK(n_root.ToGraphDef(&n_graph_def)); + + GraphDef result_graph_def; + TF_ASSERT_OK(graph_transforms::ReplaceSendRecvs( + o_graph_def, n_graph_def, {"placeholder", "placeholder_1"}, {"add"}, + &result_graph_def)); + + std::map node_map; + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(1, node_map.count("placeholder")); + EXPECT_EQ(1, node_map.count("placeholder_1")); + EXPECT_EQ(1, node_map.count("add")); + } + void TestRemoveUnusedNodes() { using namespace ::tensorflow::ops; // NOLINT(build/namespaces) auto root = tensorflow::Scope::NewRootScope(); - using namespace ::tensorflow::ops; // NOLINT(build/namespaces) const int width = 100; @@ -295,6 +328,48 @@ class ConstantFoldingTest : public ::testing::Test { EXPECT_EQ(1, node_map.count("output")); EXPECT_EQ(0, node_map.count("unused")); } + + void TestRemoveUnusedNodesMultipleOutputs() { + using namespace ::tensorflow::ops; // NOLINT(build/namespaces) + auto root = tensorflow::Scope::NewRootScope(); + + // a b + // \ / + // shape_n + // \ / + // c + auto a = Placeholder(root.WithOpName("a"), DT_FLOAT); + auto b = Placeholder(root.WithOpName("b"), DT_FLOAT); + auto shape_n = ShapeN(root.WithOpName("shape_n"), {Output(a), Output(b)}); + auto c = Add(root.WithOpName("c"), shape_n[0], shape_n[1]); + + GraphDef graph_def; + TF_ASSERT_OK(root.ToGraphDef(&graph_def)); + GraphDef result_graph_def; + TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes( + graph_def, {{shape_n[0].name()}, {"c"}}, &result_graph_def)); + + // Only one output of shape_n node is fed input. Hence the graph search + // should propagate to inputs of shape_n. Nothing to remove here. + std::map node_map; + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(1, node_map.count("a")); + EXPECT_EQ(1, node_map.count("b")); + EXPECT_EQ(1, node_map.count("c")); + + result_graph_def.Clear(); + TF_ASSERT_OK(graph_transforms::RemoveUnusedNodes( + graph_def, {{shape_n[0].name(), shape_n[1].name()}, {"c"}}, + &result_graph_def)); + + // Both outputs of shape_n node are fed inputs. shape_n does not function + // and inputs to shape_n should be removed. + node_map.clear(); + graph_transforms::MapNamesToNodes(result_graph_def, &node_map); + EXPECT_EQ(0, node_map.count("a")); + EXPECT_EQ(0, node_map.count("b")); + EXPECT_EQ(1, node_map.count("c")); + } }; TEST_F(ConstantFoldingTest, TestSimpleAdd) { TestSimpleAdd(); } @@ -309,7 +384,15 @@ TEST_F(ConstantFoldingTest, TestPreserveOutputShapes) { TEST_F(ConstantFoldingTest, TestReplaceSendRecvs) { TestReplaceSendRecvs(); } +TEST_F(ConstantFoldingTest, TestReplaceSendRecvsPrefixNames) { + TestReplaceSendRecvsPrefixNames(); +} + TEST_F(ConstantFoldingTest, TestRemoveUnusedNodes) { TestRemoveUnusedNodes(); } +TEST_F(ConstantFoldingTest, TestRemoveUnusedNodesMultipleOutputs) { + TestRemoveUnusedNodesMultipleOutputs(); +} + } // namespace graph_transforms } // namespace tensorflow diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD index 3c4e1b66bc467c7436021d2c66b8eb0a1b26cc61..c6e577223f94c9eeaff6aea9e815d7241852e391 100644 --- a/tensorflow/tools/pip_package/BUILD +++ b/tensorflow/tools/pip_package/BUILD @@ -85,6 +85,7 @@ py_binary( "//tensorflow/python:spectral_ops_test_util", "//tensorflow/python/tools:tools_pip", "//tensorflow/python/eager:eager_pip", + "//tensorflow/contrib/summary:summary_test_util", # These targets don't build on Windows yet. Exclude them for now. # "//tensorflow/contrib/ndlstm", # "//tensorflow/contrib/slim", @@ -152,6 +153,7 @@ sh_binary( "//tensorflow:tensorflow_py", "//tensorflow/contrib/boosted_trees:boosted_trees_pip", "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip", + "//tensorflow/contrib/eager/python/examples:examples_pip", "//tensorflow/contrib/gan:gan", "//tensorflow/contrib/graph_editor:graph_editor_pip", "//tensorflow/contrib/keras:keras", @@ -166,6 +168,7 @@ sh_binary( "//tensorflow/contrib/slim/python/slim/data:data_pip", "//tensorflow/contrib/slim/python/slim/nets:nets_pip", "//tensorflow/contrib/specs:specs", + "//tensorflow/contrib/summary:summary_test_util", "//tensorflow/contrib/tensor_forest:init_py", "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip", "//tensorflow/contrib/timeseries:timeseries_pip", diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 2ffaf7b1aa2a5b6d467697db59f23cdcab273be6..071b3a2a1888c39e0a22b92edbeecfaa06c8ea83 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -29,13 +29,13 @@ from setuptools.dist import Distribution # This version string is semver compatible, but incompatible with pip. # For pip, we will remove all '-' characters from this string, and use the # result for pip. -_VERSION = '1.4.0-rc0' +_VERSION = '1.4.0-rc1' REQUIRED_PACKAGES = [ 'enum34 >= 1.1.6', 'numpy >= 1.12.1', 'six >= 1.10.0', - 'protobuf >= 3.3.0', + 'protobuf >= 3.4.0', 'tensorflow-tensorboard >= 0.4.0rc1, < 0.5.0', ] @@ -67,6 +67,7 @@ if sys.version_info < (3, 4): # pylint: disable=line-too-long CONSOLE_SCRIPTS = [ + 'freeze_graph = tensorflow.python.tools.freeze_graph:main', 'saved_model_cli = tensorflow.python.tools.saved_model_cli:main', # We need to keep the TensorBoard command, even though the console script # is now declared by the tensorboard pip package. If we remove the diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b5618a06ca75398f29d1267829c40c0343b5300c..e25e12d5c5f01148221a6cb5f41aad14830dbb65 100644 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -157,7 +157,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): mkl_repository( name = "mkl", urls = [ - "http://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz", + "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz", # "https://github.com/01org/mkl-dnn/releases/download/v0.9/mklml_lnx_2018.0.20170720.tgz", ], sha256 = "57ba56c4c243f403ff78f417ff854ef50b9eddf4a610a917b7c95e7fa8553a4b", @@ -173,8 +173,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "mkl_dnn", urls = [ + "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", "https://github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", - "http://mirror.bazel.build/github.com/01org/mkl-dnn/archive/b01e3a55a07be62172e713bcd2644c5176360212.tar.gz", ], sha256 = "0d529ad4c49dc799e6df07c2b88b115d0668735da15fb3b3862d28d33fa68165", strip_prefix = "mkl-dnn-b01e3a55a07be62172e713bcd2644c5176360212", @@ -184,8 +184,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "eigen_archive", urls = [ + "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", "https://bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", - "http://mirror.bazel.build/bitbucket.org/eigen/eigen/get/429aa5254200.tar.gz", ], sha256 = "61d8b6fc4279dd1dda986fb1677d15e3d641c07a3ea5abe255790b1f0c0c14e9", strip_prefix = "eigen-eigen-429aa5254200", @@ -198,7 +198,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "970285762565c7890c6c087d262b0a18286e7d0384f13a37786d8521773bc969", strip_prefix = "tools-0e906ebc527eab1cdbf7adabff5b474da9562e9f/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf", urls = [ - "http://mirror.bazel.build/github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz", + "https://mirror.bazel.build/github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz", # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz", ], ) @@ -206,7 +206,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "libxsmm_archive", urls = [ - "http://mirror.bazel.build/github.com/hfp/libxsmm/archive/1.8.1.tar.gz", + "https://mirror.bazel.build/github.com/hfp/libxsmm/archive/1.8.1.tar.gz", # "https://github.com/hfp/libxsmm/archive/1.8.1.tar.gz", ], sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce", @@ -222,7 +222,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "ortools_archive", urls = [ - "http://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz", + "https://mirror.bazel.build/github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz", # "https://github.com/google/or-tools/archive/253f7955c6a1fd805408fba2e42ac6d45b312d15.tar.gz", ], sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755", @@ -233,7 +233,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_googlesource_code_re2", urls = [ - "http://mirror.bazel.build/github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz", + "https://mirror.bazel.build/github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz", # "https://github.com/google/re2/archive/b94b7cd42e9f02673cd748c1ac1d16db4052514c.tar.gz", ], sha256 = "bd63550101e056427c9e7ff12a408c1c8b74e9803f393ca916b2926fc2c4906f", @@ -243,7 +243,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "gemmlowp", urls = [ - "http://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip" + "https://mirror.bazel.build/github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip" # "https://github.com/google/gemmlowp/archive/010bb3e71a26ca1d0884a167081d092b43563996.zip", ], sha256 = "dd2557072bde12141419cb8320a9c25e6ec41a8ae53c2ac78c076a347bb46d9d", @@ -253,7 +253,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "farmhash_archive", urls = [ - "http://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz", + "https://mirror.bazel.build/github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz", # "https://github.com/google/farmhash/archive/816a4ae622e964763ca0862d9dbd19324a1eaf45.tar.gz", ], sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0", @@ -269,7 +269,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "highwayhash", urls = [ - "http://mirror.bazel.build/github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz", + "https://mirror.bazel.build/github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz", # "https://github.com/google/highwayhash/archive/dfcb97ca4fe9277bf9dc1802dd979b071896453b.tar.gz", ], sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9", @@ -280,7 +280,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "nasm", urls = [ - "http://mirror.bazel.build/www.nasm.us/pub/nasm/releasebuilds/2.12.02/nasm-2.12.02.tar.bz2", + "https://mirror.bazel.build/www.nasm.us/pub/nasm/releasebuilds/2.12.02/nasm-2.12.02.tar.bz2", "http://pkgs.fedoraproject.org/repo/pkgs/nasm/nasm-2.12.02.tar.bz2/d15843c3fb7db39af80571ee27ec6fad/nasm-2.12.02.tar.bz2", ], sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324", @@ -291,7 +291,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "jpeg", urls = [ - "http://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", + "https://mirror.bazel.build/github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", # "https://github.com/libjpeg-turbo/libjpeg-turbo/archive/1.5.1.tar.gz", ], sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77", @@ -303,7 +303,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "png_archive", urls = [ - "http://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.2.53.tar.gz", + "https://mirror.bazel.build/github.com/glennrp/libpng/archive/v1.2.53.tar.gz", # "https://github.com/glennrp/libpng/archive/v1.2.53.tar.gz", ], sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2", @@ -314,7 +314,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "sqlite_archive", urls = [ - "http://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", + "https://mirror.bazel.build/www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", "http://www.sqlite.org/2017/sqlite-amalgamation-3200000.zip", ], sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4", @@ -325,7 +325,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "gif_archive", urls = [ - "http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz", + "https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz", "http://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.1.4.tar.gz", ], sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1", @@ -336,7 +336,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "six_archive", urls = [ - "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz", + "https://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz", "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz", ], sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a", @@ -347,7 +347,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "org_python_pypi_backports_weakref", urls = [ - "http://mirror.bazel.build/pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz", + "https://mirror.bazel.build/pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz", "https://pypi.python.org/packages/bc/cc/3cdb0a02e7e96f6c70bd971bc8a90b8463fda83e264fa9c5c1c98ceabd81/backports.weakref-1.0rc1.tar.gz", ], sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892", @@ -358,7 +358,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "com_github_andreif_codegen", urls = [ - "http://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz", + "https://mirror.bazel.build/github.com/andreif/codegen/archive/1.0.tar.gz", # "https://github.com/andreif/codegen/archive/1.0.tar.gz", ], sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee", @@ -371,7 +371,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): licenses = ["notice"], # Python 2.0 sha256_urls = { "b5556e921715ddb9242c076cae3963f483aa47266c5e37ea4c187f77cc79501c": [ - "http://mirror.bazel.build/docs.python.org/2.7/_sources/license.txt", + "https://mirror.bazel.build/docs.python.org/2.7/_sources/license.txt", "https://docs.python.org/2.7/_sources/license.txt", ], }, @@ -387,7 +387,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): patched_http_archive( name = "protobuf_archive", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/b04e5cba356212e4e8c66c61bbe0c3a20537c5b9.tar.gz", ], sha256 = "e178a25c52efcb6b05988bdbeace4c0d3f2d2fe5b46696d1d9898875c3803d6a", strip_prefix = "protobuf-b04e5cba356212e4e8c66c61bbe0c3a20537c5b9", @@ -410,7 +410,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_google_protobuf", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", @@ -420,7 +420,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_google_protobuf_cc", urls = [ - "http://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", + "https://mirror.bazel.build/github.com/google/protobuf/archive/0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66.tar.gz", ], sha256 = "6d43b9d223ce09e5d4ce8b0060cb8a7513577a35a64c7e3dad10f0703bf3ad93", strip_prefix = "protobuf-0b059a3d8a8f8aa40dde7bea55edca4ec5dfea66", @@ -429,17 +429,17 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "nsync", urls = [ - "http://mirror.bazel.build/github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz", - # "https://github.com/google/nsync/archive/ad722c76c6e6653f66be2e1f69521b7f7517da55.tar.gz", + "https://mirror.bazel.build/github.com/google/nsync/archive/839fcc53ff9be58218ed55397deb3f8376a1444e.tar.gz", + # "https://github.com/google/nsync/archive/839fcc53ff9be58218ed55397deb3f8376a1444e.tar.gz", ], - sha256 = "7dd8ca49319f77e8226cd020a9210a525f88ac26e7041c59c95418223a1cdf55", - strip_prefix = "nsync-ad722c76c6e6653f66be2e1f69521b7f7517da55", + sha256 = "124d105edb0313ef2d7f5bb86ec94d9f8de95479e55641c4254ffa8f795e9b37", + strip_prefix = "nsync-839fcc53ff9be58218ed55397deb3f8376a1444e", ) native.http_archive( name = "com_google_googletest", urls = [ - "http://mirror.bazel.build/github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip", + "https://mirror.bazel.build/github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip", # "https://github.com/google/googletest/archive/9816b96a6ddc0430671693df90192bbee57108b6.zip", ], sha256 = "9cbca84c4256bed17df2c8f4d00c912c19d247c11c9ba6647cd6dd5b5c996b8d", @@ -449,7 +449,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "com_github_gflags_gflags", urls = [ - "http://mirror.bazel.build/github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz", + "https://mirror.bazel.build/github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz", # "https://github.com/gflags/gflags/archive/f8a0efe03aa69b3336d8e228b37d4ccb17324b88.tar.gz", ], sha256 = "4d222fab8f1ede4709cdff417d15a1336f862d7334a81abf76d09c15ecf9acd1", @@ -465,7 +465,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "pcre", sha256 = "ccdf7e788769838f8285b3ee672ed573358202305ee361cfec7a4a4fb005bbc7", urls = [ - "http://mirror.bazel.build/ftp.exim.org/pub/pcre/pcre-8.39.tar.gz", + "https://mirror.bazel.build/ftp.exim.org/pub/pcre/pcre-8.39.tar.gz", "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz", ], strip_prefix = "pcre-8.39", @@ -476,7 +476,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "swig", sha256 = "58a475dbbd4a4d7075e5fe86d4e54c9edde39847cdb96a3053d87cb64a23a453", urls = [ - "http://mirror.bazel.build/ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", + "https://mirror.bazel.build/ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", "http://ufpr.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz", ], @@ -488,7 +488,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "curl", sha256 = "ff3e80c1ca6a068428726cd7dd19037a47cc538ce58ef61c59587191039b2ca6", urls = [ - "http://mirror.bazel.build/curl.haxx.se/download/curl-7.49.1.tar.gz", + "https://mirror.bazel.build/curl.haxx.se/download/curl-7.49.1.tar.gz", "https://curl.haxx.se/download/curl-7.49.1.tar.gz", ], strip_prefix = "curl-7.49.1", @@ -518,7 +518,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): patched_http_archive( name = "grpc", urls = [ - "http://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz", + "https://mirror.bazel.build/github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz", # "https://github.com/grpc/grpc/archive/781fd6f6ea03645a520cd5c675da67ab61f87e4b.tar.gz", ], sha256 = "2004635e6a078acfac8ffa71738397796be4f8fb72f572cc44ecee5d99511d9f", @@ -542,7 +542,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "linenoise", sha256 = "7f51f45887a3d31b4ce4fa5965210a5e64637ceac12720cfce7954d6a2e812f7", urls = [ - "http://mirror.bazel.build/github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz", + "https://mirror.bazel.build/github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz", # "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz", ], strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3", @@ -554,7 +554,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "llvm", urls = [ - "http://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz", + "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz", "https://github.com/llvm-mirror/llvm/archive/bb3c660e87f59abb665570a31b01ab125ec4c10e.tar.gz", ], sha256 = "caab6d7978e6771cb4e9b5b89607c5370de8aa642913c6c14e892468194c94e4", @@ -566,7 +566,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "lmdb", urls = [ - "http://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz", + "https://mirror.bazel.build/github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz", # "https://github.com/LMDB/lmdb/archive/LMDB_0.9.19.tar.gz", ], sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326", @@ -577,7 +577,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "jsoncpp_git", urls = [ - "http://mirror.bazel.build/github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz", + "https://mirror.bazel.build/github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz", # "https://github.com/open-source-parsers/jsoncpp/archive/11086dd6a7eba04289944367ca82cea71299ed70.tar.gz", ], sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2", @@ -590,21 +590,19 @@ def tf_workspace(path_prefix="", tf_repo_name=""): actual = "@jsoncpp_git//:jsoncpp", ) - patched_http_archive( + native.http_archive( name = "boringssl", urls = [ - "https://github.com/google/boringssl/archive/72cfd9f49ec5fbc2db368b76398c196dafe6a4bc.tar.gz", + "https://mirror.bazel.build/github.com/google/boringssl/archive/a0fb951d2a26a8ee746b52f3ba81ab011a0af778.tar.gz", ], - sha256 = "5e6f7b72c74adeb902581271925ddb979e77b96327abd76604ce894d80680e51", - strip_prefix = "boringssl-72cfd9f49ec5fbc2db368b76398c196dafe6a4bc", - # Add patch to boringssl code to support s390x - patch_file = str(Label("//third_party/boringssl:add_boringssl_s390x.patch")), + sha256 = "524ba98a56300149696481b4cb9ddebd0c7b7ac9b9f6edee81da2d2d7e5d2bb3", + strip_prefix = "boringssl-a0fb951d2a26a8ee746b52f3ba81ab011a0af778", ) native.new_http_archive( name = "zlib_archive", urls = [ - "http://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz", + "https://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz", "http://zlib.net/fossils/zlib-1.2.8.tar.gz", ], sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d", @@ -620,7 +618,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "fft2d", urls = [ - "http://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz", + "https://mirror.bazel.build/www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz", "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz", ], sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296", @@ -630,7 +628,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "snappy", urls = [ - "http://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz", + "https://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz", # "https://github.com/google/snappy/archive/1.1.4.tar.gz", ], sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94", @@ -642,7 +640,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "nccl_archive", urls = [ - "http://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", + "https://mirror.bazel.build/github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", # "https://github.com/nvidia/nccl/archive/03d856977ecbaac87e598c0c4bafca96761b9ac7.tar.gz", ], sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176", @@ -660,13 +658,14 @@ def tf_workspace(path_prefix="", tf_repo_name=""): sha256 = "f599b57aec4f03ad696044dd430b2d201864113937353adc346f53ad47991319", strip_prefix = "aws-sdk-cpp-1.0.90", build_file = str(Label("//third_party:aws.BUILD")), + repository = tf_repo_name ) java_import_external( name = "junit", jar_sha256 = "59721f0805e223d84b90677887d9ff567dc534d7c502ca903c0c2b17f05c116a", jar_urls = [ - "http://mirror.bazel.build/repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar", + "https://mirror.bazel.build/repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar", "http://repo1.maven.org/maven2/junit/junit/4.12/junit-4.12.jar", "http://maven.ibiblio.org/maven2/junit/junit/4.12/junit-4.12.jar", ], @@ -679,7 +678,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "org_hamcrest_core", jar_sha256 = "66fdef91e9739348df7a096aa384a5685f4e875584cce89386a7a47251c4d8e9", jar_urls = [ - "http://mirror.bazel.build/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", + "https://mirror.bazel.build/repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", "http://repo1.maven.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", "http://maven.ibiblio.org/maven2/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar", ], @@ -690,7 +689,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): temp_workaround_http_archive( name = "jemalloc", urls = [ - "http://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", + "https://mirror.bazel.build/github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", # "https://github.com/jemalloc/jemalloc/archive/4.4.0.tar.gz", ], sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8", @@ -724,10 +723,20 @@ def tf_workspace(path_prefix="", tf_repo_name=""): deps = ["@com_google_guava"], ) + java_import_external( + name = "javax_validation", + jar_sha256 = "e459f313ebc6db2483f8ceaad39af07086361b474fa92e40f442e8de5d9895dc", + jar_urls = [ + "http://mirror.bazel.build/repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar", + "http://repo1.maven.org/maven2/javax/validation/validation-api/1.0.0.GA/validation-api-1.0.0.GA.jar", + ], + licenses = ["notice"], # Apache 2.0 + ) + native.new_http_archive( name = "com_google_pprof", urls = [ - "http://mirror.bazel.build/github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz", + "https://mirror.bazel.build/github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz", # "https://github.com/google/pprof/archive/c0fb62ec88c411cc91194465e54db2632845b650.tar.gz", ], sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4", @@ -738,8 +747,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.new_http_archive( name = "cub_archive", urls = [ - "http://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip", - "https://github.com/NVlabs/cub/archive/1.7.4.zip", + "https://mirror.bazel.build/github.com/NVlabs/cub/archive/1.7.4.zip", + # "https://github.com/NVlabs/cub/archive/1.7.4.zip", ], sha256 = "20a1a39fd97e5da7f40f5f2e7fd73fd2ea59f9dc4bb8a6c5f228aa543e727e31", strip_prefix = "cub-1.7.4", @@ -755,7 +764,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""): name = "cython", sha256 = "6dcd30b5ceb887b2b965ee7ceb82ea3acb5f0642fe2206c7636b45acea4798e5", urls = [ - "http://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz", + "https://mirror.bazel.build/github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz", "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz", ], strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17", @@ -765,9 +774,31 @@ def tf_workspace(path_prefix="", tf_repo_name=""): native.http_archive( name = "bazel_toolchains", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", - # "https://github.com/bazelbuild/bazel-toolchains/archive/b2b4b38433bf2d1159360855ea4004378308711b.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz", + # "https://github.com/bazelbuild/bazel-toolchains/archive/af4681c3d19f063f090222ec3d04108c4e0ca255.tar.gz", + ], + sha256 = "d58bb2d6c8603f600d522b6104d6192a65339aa26cbba9f11ff5c4b36dedb928", + strip_prefix = "bazel-toolchains-af4681c3d19f063f090222ec3d04108c4e0ca255", + ) + + native.new_http_archive( + name = "arm_neon_2_x86_sse", + sha256 = "c8d90aa4357f8079d427e87a6f4c493da1fa4140aee926c05902d7ec1533d9a5", + strip_prefix = "ARM_NEON_2_x86_SSE-0f77d9d182265259b135dad949230ecbf1a2633d", + urls = [ + "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", + "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz", + ], + build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")), + ) + + native.new_http_archive( + name = "flatbuffers", + build_file = "third_party/flatbuffers/flatbuffers.BUILD", + strip_prefix = "flatbuffers-971a68110e4fc1bace10fcb6deeb189e7e1a34ce", + sha256 = "874088d2ee0d9f8524191f77209556415f03dd44e156276edf19e5b90ceb5f55", + urls = [ + "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz", + "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz", ], - sha256 = "46187270ca04ff8109980f45c3438fabfe48695e163789096eb82ee097ffe685", - strip_prefix = "bazel-toolchains-b2b4b38433bf2d1159360855ea4004378308711b", ) diff --git a/third_party/arm_neon_2_x86_sse.BUILD b/third_party/arm_neon_2_x86_sse.BUILD new file mode 100644 index 0000000000000000000000000000000000000000..6c641a7f4e2301ede943779b0f237dd8b75aa6dd --- /dev/null +++ b/third_party/arm_neon_2_x86_sse.BUILD @@ -0,0 +1,16 @@ +# Description: +# NEON2SSE - a header file redefining ARM Neon intrinsics in terms of SSE intrinsics +# allowing neon code to compile and run on x64/x86 workstantions. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # 3-Clause BSD + +exports_files([ + "LICENSE", +]) + +cc_library( + name = "arm_neon_2_x86_sse", + hdrs = ["NEON_2_SSE.h"], +) diff --git a/third_party/aws.BUILD b/third_party/aws.BUILD index 9d8e7946cd5d00263e08cac126a8483b0a91ea8e..bc6a2fd8cc6b8db251a026749daef9c0f6e875f5 100644 --- a/third_party/aws.BUILD +++ b/third_party/aws.BUILD @@ -18,6 +18,9 @@ cc_library( "@%ws%//tensorflow:darwin": glob([ "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", ]), + "@%ws%//tensorflow:linux_ppc64le": glob([ + "aws-cpp-sdk-core/source/platform/linux-shared/*.cpp", + ]), "//conditions:default": [], }) + glob([ "aws-cpp-sdk-core/include/**/*.h", @@ -57,6 +60,11 @@ cc_library( "ENABLE_CURL_CLIENT", "ENABLE_NO_ENCRYPTION", ], + "@%ws%//tensorflow:linux_ppc64le": [ + "PLATFORM_LINUX", + "ENABLE_CURL_CLIENT", + "ENABLE_NO_ENCRYPTION", + ], "//conditions:default": [], }), includes = [ diff --git a/third_party/boringssl/add_boringssl_s390x.patch b/third_party/boringssl/add_boringssl_s390x.patch deleted file mode 100644 index b684dc6df773e481237c45ca8fbc0eb48bfb0746..0000000000000000000000000000000000000000 --- a/third_party/boringssl/add_boringssl_s390x.patch +++ /dev/null @@ -1,23 +0,0 @@ -diff -ur a/BUILD b/BUILD ---- a/BUILD 2017-10-10 15:50:34.000000000 +0000 -+++ b/BUILD 2017-10-15 21:19:02.057606476 +0000 -@@ -63,6 +63,7 @@ - "-Wwrite-strings", - "-Wshadow", - "-fno-common", -+ "-Wno-uninitialized", - - # Modern build environments should be able to set this to use atomic - # operations for reference counting rather than locks. However, it's -diff -ur a/src/include/openssl/base.h b/src/include/openssl/base.h ---- a/src/include/openssl/base.h 2017-10-10 15:50:34.000000000 +0000 -+++ b/src/include/openssl/base.h 2017-10-15 19:49:38.182154627 +0000 -@@ -106,6 +106,8 @@ - #define OPENSSL_PNACL - #elif defined(__myriad2__) - #define OPENSSL_32_BIT -+#elif defined(__s390x__) -+#define OPENSSL_64_BIT - #else - // Note BoringSSL only supports standard 32-bit and 64-bit two's-complement, - // little-endian architectures. Functions will not produce the correct answer diff --git a/third_party/eigen3/BUILD b/third_party/eigen3/BUILD index f38a26717e14ea727953e1e702f2fdeb45158fb4..ad87477b7aa304581c9164d3d10574c1069f03cc 100644 --- a/third_party/eigen3/BUILD +++ b/third_party/eigen3/BUILD @@ -38,3 +38,12 @@ cc_library( "@local_config_sycl//sycl:sycl", ], ) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/third_party/fft2d/BUILD b/third_party/fft2d/BUILD index 93ea06e81b85d3ffca90133225604e9ac3a44333..813544248269ea38a86dfb0d942c0a98319be078 100644 --- a/third_party/fft2d/BUILD +++ b/third_party/fft2d/BUILD @@ -28,3 +28,12 @@ filegroup( name = "fft2d_headers_srcs", srcs = ["fft.h"], ) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = ["**/OWNERS"], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/third_party/flatbuffers/BUILD b/third_party/flatbuffers/BUILD new file mode 100644 index 0000000000000000000000000000000000000000..fbdf19f2054cf01aec44e3fcb13d0d0a2ff6f914 --- /dev/null +++ b/third_party/flatbuffers/BUILD @@ -0,0 +1,15 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/third_party/flatbuffers/build_defs.bzl b/third_party/flatbuffers/build_defs.bzl new file mode 100644 index 0000000000000000000000000000000000000000..ae8d7feebe781c896a408dbc7119a4f0820d0519 --- /dev/null +++ b/third_party/flatbuffers/build_defs.bzl @@ -0,0 +1,196 @@ +# Description: +# BUILD rules for generating flatbuffer files. + +flatc_path = "@flatbuffers//:flatc" + +DEFAULT_FLATC_ARGS = [ + "--no-union-value-namespacing", + "--gen-object-api", +] + +def flatbuffer_library_public(name, + srcs, + outs, + language_flag, + out_prefix="", + includes=[], + include_paths=[], + flatc_args=DEFAULT_FLATC_ARGS, + reflection_name="", + reflection_visiblity=None, + output_to_bindir=False): + '''Generates code files for reading/writing the given flatbuffers in the requested language using the public compiler. + + Args: + name: Rule name. + srcs: Source .fbs files. Sent in order to the compiler. + outs: Output files from flatc. + language_flag: Target language flag. One of [-c, -j, -js]. + out_prefix: Prepend this path to the front of all generated files except on + single source targets. Usually is a directory name. + includes: Optional, list of filegroups of schemas that the srcs depend on. + include_paths: Optional, list of paths the includes files can be found in. + flatc_args: Optional, list of additional arguments to pass to flatc. + reflection_name: Optional, if set this will generate the flatbuffer + reflection binaries for the schemas. + reflection_visiblity: The visibility of the generated reflection Fileset. + output_to_bindir: Passed to genrule for output to bin directory. + Outs: + filegroup(name): all generated source files. + Fileset([reflection_name]): (Optional) all generated reflection binaries. + ''' + include_paths_cmd = ["-I %s" % (s) for s in include_paths] + # '$(@D)' when given a single source target will give the appropriate + # directory. Appending 'out_prefix' is only necessary when given a build + # target with multiple sources. + output_directory = ( + ("-o $(@D)/%s" % (out_prefix)) if len(srcs) > 1 else ("-o $(@D)")) + genrule_cmd = " ".join([ + "for f in $(SRCS); do", + "$(location %s)" % (flatc_path), + " ".join(flatc_args), + " ".join(include_paths_cmd), + language_flag, + output_directory, + "$$f;", + "done", + ]) + native.genrule( + name=name, + srcs=srcs, + outs=outs, + output_to_bindir=output_to_bindir, + tools=includes + [flatc_path,], + cmd=genrule_cmd, + message="Generating flatbuffer files for %s:" % (name),) + if reflection_name: + reflection_genrule_cmd = " ".join([ + "for f in $(SRCS); do", + "$(location %s)" % (flatc_path), + "-b --schema", + " ".join(flatc_args), + " ".join(include_paths_cmd), + language_flag, + output_directory, + "$$f;", + "done", + ]) + reflection_outs = [ + (out_prefix + "%s.bfbs") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs + ] + native.genrule( + name= "%s_srcs" % reflection_name, + srcs=srcs, + outs=reflection_outs, + output_to_bindir=output_to_bindir, + tools=includes + [flatc_path,], + cmd=reflection_genrule_cmd, + message="Generating flatbuffer reflection binary for %s:" % (name),) + native.Fileset( + name=reflection_name, + out="%s_out" % reflection_name, + entries=[ + native.FilesetEntry(files=reflection_outs), + ], + visibility=reflection_visiblity + ) + + +def flatbuffer_cc_library(name, srcs, srcs_filegroup_name="", + out_prefix="", includes=[], include_paths=[], + flatc_args=DEFAULT_FLATC_ARGS, + visibility=None, srcs_filegroup_visibility=None, + gen_reflections=False): + '''A cc_library with the generated reader/writers for the given flatbuffer definitions. + + Args: + name: Rule name. + srcs: Source .fbs files. Sent in order to the compiler. + srcs_filegroup_name: Name of the output filegroup that holds srcs. Pass this + filegroup into the `includes` parameter of any other + flatbuffer_cc_library that depends on this one's schemas. + out_prefix: Prepend this path to the front of all generated files. Usually + is a directory name. + includes: Optional, list of filegroups of schemas that the srcs depend on. + ** SEE REMARKS BELOW ** + include_paths: Optional, list of paths the includes files can be found in. + flatc_args: Optional list of additional arguments to pass to flatc + (e.g. --gen-mutable). + visibility: The visibility of the generated cc_library. By default, use the + default visibility of the project. + srcs_filegroup_visibility: The visibility of the generated srcs filegroup. + By default, use the value of the visibility parameter above. + gen_reflections: Optional, if true this will generate the flatbuffer + reflection binaries for the schemas. + Outs: + filegroup([name]_srcs): all generated .h files. + filegroup(srcs_filegroup_name if specified, or [name]_includes if not): + Other flatbuffer_cc_library's can pass this in for their `includes` + parameter, if they depend on the schemas in this library. + Fileset([name]_reflection): (Optional) all generated reflection binaries. + cc_library([name]): library with sources and flatbuffers deps. + + Remarks: + ** Because the genrule used to call flatc does not have any trivial way of + computing the output list of files transitively generated by includes and + --gen-includes (the default) being defined for flatc, the --gen-includes + flag will not work as expected. The way around this is to add a dependency + to the flatbuffer_cc_library defined alongside the flatc included Fileset. + For example you might define: + + flatbuffer_cc_library( + name = "my_fbs", + srcs = [ "schemas/foo.fbs" ], + includes = [ "//third_party/bazz:bazz_fbs_includes" ], + ) + + In which foo.fbs includes a few files from the Fileset defined at + //third_party/bazz:bazz_fbs_includes. When compiling the library that + includes foo_generated.h, and therefore has my_fbs as a dependency, it + will fail to find any of the bazz *_generated.h files unless you also + add bazz's flatbuffer_cc_library to your own dependency list, e.g.: + + cc_library( + name = "my_lib", + deps = [ + ":my_fbs", + "//third_party/bazz:bazz_fbs" + ], + ) + + Happy dependent Flatbuffering! + ''' + output_headers = [ + (out_prefix + "%s_generated.h") % (s.replace(".fbs", "").split("/")[-1]) for s in srcs + ] + reflection_name = "%s_reflection" % name if gen_reflections else "" + + flatbuffer_library_public(name="%s_srcs" % (name), + srcs=srcs, + outs=output_headers, + language_flag="-c", + out_prefix=out_prefix, + includes=includes, + include_paths=include_paths, + flatc_args=flatc_args, + reflection_name=reflection_name, + reflection_visiblity=visibility,) + native.cc_library(name=name, + hdrs=output_headers, + srcs=output_headers, + features=[ + "-parse_headers", + ], + deps=[ + "@flatbuffers//:runtime_cc", + ], + includes=["."], + linkstatic=1, + visibility=visibility) + + # A filegroup for the `srcs`. That is, all the schema files for this + # Flatbuffer set. + native.filegroup( + name = srcs_filegroup_name if srcs_filegroup_name else "%s_includes" % (name), + srcs = srcs, + visibility=srcs_filegroup_visibility if srcs_filegroup_visibility != None else visibility) diff --git a/third_party/flatbuffers/flatbuffers.BUILD b/third_party/flatbuffers/flatbuffers.BUILD new file mode 100644 index 0000000000000000000000000000000000000000..a426db0c5027dc27cec4c5587ddb0990d60f1d6e --- /dev/null +++ b/third_party/flatbuffers/flatbuffers.BUILD @@ -0,0 +1,127 @@ +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) # Apache 2.0 + +FLATBUFFERS_COPTS = [ + "-fexceptions", + "-Wno-implicit-fallthrough", +] + +# Public flatc library to compile flatbuffer files at runtime. +cc_library( + name = "flatbuffers", + srcs = [ + "include/flatbuffers/code_generators.h", + "include/flatbuffers/reflection_generated.h", + "src/code_generators.cpp", + "src/idl_gen_fbs.cpp", + "src/idl_gen_general.cpp", + "src/idl_gen_text.cpp", + "src/idl_parser.cpp", + "src/reflection.cpp", + "src/util.cpp", + ], + hdrs = [ + "include/flatbuffers/base.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/flexbuffers.h", + "include/flatbuffers/hash.h", + "include/flatbuffers/idl.h", + "include/flatbuffers/reflection.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + ], + copts = FLATBUFFERS_COPTS, + includes = ["include/"], +) + +# Public flatc compiler library. +cc_library( + name = "flatc_library", + srcs = [ + "grpc/src/compiler/config.h", + "grpc/src/compiler/go_generator.h", + "grpc/src/compiler/schema_interface.h", + "include/flatbuffers/base.h", + "include/flatbuffers/code_generators.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/flatc.h", + "include/flatbuffers/flexbuffers.h", + "include/flatbuffers/hash.h", + "include/flatbuffers/idl.h", + "include/flatbuffers/reflection.h", + "include/flatbuffers/reflection_generated.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + "src/code_generators.cpp", + "src/flatc.cpp", + "src/idl_gen_fbs.cpp", + "src/idl_parser.cpp", + "src/reflection.cpp", + "src/util.cpp", + ], + hdrs = [ + "include/flatbuffers/base.h", + "include/flatbuffers/code_generators.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/flatc.h", + "include/flatbuffers/idl.h", + "include/flatbuffers/reflection.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + ], + copts = FLATBUFFERS_COPTS, + includes = [ + "grpc/", + "include/", + ], +) + +# Public flatc compiler. +cc_binary( + name = "flatc", + srcs = [ + "grpc/src/compiler/cpp_generator.cc", + "grpc/src/compiler/cpp_generator.h", + "grpc/src/compiler/go_generator.cc", + "grpc/src/compiler/go_generator.h", + "grpc/src/compiler/schema_interface.h", + "src/flatc_main.cpp", + "src/idl_gen_cpp.cpp", + "src/idl_gen_general.cpp", + "src/idl_gen_go.cpp", + "src/idl_gen_grpc.cpp", + "src/idl_gen_js.cpp", + "src/idl_gen_json_schema.cpp", + "src/idl_gen_php.cpp", + "src/idl_gen_python.cpp", + "src/idl_gen_text.cpp", + ], + copts = FLATBUFFERS_COPTS, + includes = [ + "grpc/", + "include/", + ], + deps = [ + ":flatc_library", + ], +) + +filegroup( + name = "runtime_cc_srcs", + srcs = [ + "include/flatbuffers/base.h", + "include/flatbuffers/flatbuffers.h", + "include/flatbuffers/stl_emulation.h", + "include/flatbuffers/util.h", + ], +) + +cc_library( + name = "runtime_cc", + hdrs = ["runtime_cc_srcs"], + includes = ["include"], + linkstatic = 1, +) diff --git a/third_party/toolchains/gpus/crosstool/CROSSTOOL b/third_party/toolchains/gpus/crosstool/CROSSTOOL index 224b8912f6d743ad78b0ce835fdb8aa30e5e1309..a47e0c7cd74edcea777d76854c2d7e97d69897fa 100644 --- a/third_party/toolchains/gpus/crosstool/CROSSTOOL +++ b/third_party/toolchains/gpus/crosstool/CROSSTOOL @@ -296,7 +296,7 @@ toolchain { cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu/c++/5.4.0" cxx_builtin_include_directory: "/usr/include/c++/5.4.0/backward" cxx_builtin_include_directory: "/usr/local/include" - cxx_builtin_include_directory: "/usr/local/lib/clang/5.0.0/include" + cxx_builtin_include_directory: "/usr/local/lib/clang/6.0.0/include" cxx_builtin_include_directory: "/usr/include/x86_64-linux-gnu" cxx_builtin_include_directory: "/usr/include" } diff --git a/third_party/toolchains/gpus/cuda/BUILD b/third_party/toolchains/gpus/cuda/BUILD index 36be86cd1021188eccf2f8d16e17c97531a9e09a..39136de99c901d6d6a9dafefe3163972511ec122 100644 --- a/third_party/toolchains/gpus/cuda/BUILD +++ b/third_party/toolchains/gpus/cuda/BUILD @@ -1347,7 +1347,7 @@ genrule( "cuda/lib/libcupti.so.8.0", ], cmd = """ -if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart.so.8.0.61" "$(@D)/cuda/lib/libcudart.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcublas.so.8.0.71" "$(@D)/cuda/lib/libcublas.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcusolver.so.8.0.61" "$(@D)/cuda/lib/libcusolver.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcurand.so.8.0.61" "$(@D)/cuda/lib/libcurand.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcufft.so.8.0.61" "$(@D)/cuda/lib/libcufft.so.8.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.6.0.21" "$(@D)/cuda/lib/libcudnn.so.6" && cp "/usr/local/cuda-8.0/extras/CUPTI/lib64/libcupti.so.8.0.61" "$(@D)/cuda/lib/libcupti.so.8.0" +if [ -d "$(@D)/extras" ]; then rm $(@D)/extras -drf; fi && if [ -d "$(@D)/include" ]; then rm $(@D)/include -drf; fi && if [ -d "$(@D)/lib" ]; then rm $(@D)/lib -drf; fi && if [ -d "$(@D)/nvvm" ]; then rm $(@D)/nvvm -drf; fi && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/stubs/libcuda.so" "$(@D)/cuda/lib/libcuda.so" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart.so.8.0.61" "$(@D)/cuda/lib/libcudart.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcudart_static.a" "$(@D)/cuda/lib/libcudart_static.a" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcublas.so.8.0.88" "$(@D)/cuda/lib/libcublas.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcusolver.so.8.0.61" "$(@D)/cuda/lib/libcusolver.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcurand.so.8.0.61" "$(@D)/cuda/lib/libcurand.so.8.0" && cp "/usr/local/cuda-8.0/targets/x86_64-linux/lib/libcufft.so.8.0.61" "$(@D)/cuda/lib/libcufft.so.8.0" && cp "/usr/lib/x86_64-linux-gnu/libcudnn.so.6.0.21" "$(@D)/cuda/lib/libcudnn.so.6" && cp "/usr/local/cuda-8.0/extras/CUPTI/lib64/libcupti.so.8.0.61" "$(@D)/cuda/lib/libcupti.so.8.0" """, )
Version:CPU/GPU:Python Version:Compiler:Build Tools:cuDNN:CUDA:
tensorflow-1.4.0rc0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.4.0rc0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.368
tensorflow-1.4.0rc1CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.4.0rc1GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.368
tensorflow-1.2.0CPU3.5-3.6MSVC 2015 update 3Cmake v3.6.3N/AN/A
tensorflow_gpu-1.2.0GPU3.5-3.6MSVC 2015 update 3Cmake v3.6.35.18
tensorflow-1.1.0CPU3.5MSVC 2015 update 3Cmake v3.6.3N/AN/A